diff --git a/README.md b/README.md index 1cc0f80..899bca1 100644 --- a/README.md +++ b/README.md @@ -275,6 +275,26 @@ For detailed instructions on setting up the server and connecting your favorite +### Streamlit App + +The `intugle` library includes a Streamlit application that provides an interactive web interface for building and visualizing semantic data models. + +To use the Streamlit app, install `intugle` with the `streamlit` extra: + +```bash +pip install intugle[streamlit] +``` + +You can launch the Streamlit application using the `intugle-mcp` command or `uvx`: + +```bash +intugle-streamlit +# Or using uvx +uvx --from intugle intugle-streamlit +``` + +Open the URL provided in your terminal (usually `http://localhost:8501`) to access the application. For more details, refer to the [Streamlit App documentation](https://intugle.github.io/data-tools/docs/streamlit-app). + ## Community Join our community to ask questions, share your projects, and connect with other users. diff --git a/docsite/docs/streamlit-app.md b/docsite/docs/streamlit-app.md new file mode 100644 index 0000000..baff345 --- /dev/null +++ b/docsite/docs/streamlit-app.md @@ -0,0 +1,66 @@ +--- +sidebar_position: 8 +title: Streamlit App +--- + +# Intugle - Streamlit App + +This Streamlit application provides an interactive web interface for the `intugle` library. It allows users to upload their tabular data (CSV/Excel), configure a Large Language Model (LLM), and step through the process of building a semantic data model. The app profiles the data, generates a business glossary, identifies relationships between datasets, and visualizes the resulting semantic graph. + +## ✨ Features + +- **File Upload**: Upload multiple CSV or Excel files directly in the browser. +- **Interactive Data Prep**: Interactively rename tables and select, rename, or drop columns before processing. +- **LLM Configuration**: Securely configure and connect to your preferred LLM provider (OpenAI, Azure OpenAI, Gemini). +- **Automated Data Profiling**: Automatically calculates key metrics like uniqueness, completeness, and data types for every column. +- **AI-Powered Business Glossary**: Leverages an LLM to generate a business glossary for all tables and columns, adding crucial context. +- **Automated Link Prediction**: Discovers potential relationships (foreign keys) between your tables. +- **Interactive Visualization**: Displays the final semantic model as an interactive network graph. +- **Detailed Results**: Provides a tabular view of all predicted links with detailed metrics. +- **Export Artifacts**: Download the generated semantic model artifacts (`.yml` files) as a ZIP archive for use in other systems. + +## 🚀 Getting Started + +Follow these instructions to set up and run the application on your local machine. + +### Prerequisites + +- Python 3.10+ +- `uv` (Optional: for `uvx` command) + +### 1. Installation + +To use the Streamlit app, install `intugle` with the `streamlit` extra: + +```bash +pip install intugle[streamlit] +``` + +### 2. Configuration + +The application requires credentials for a Large Language Model to generate the business glossary and perform other AI-powered tasks. + +You can configure your LLM provider and API keys directly in the application's sidebar after launching it. The app will guide you on which credentials are required for your chosen provider (e.g., `OPENAI_API_KEY` for OpenAI). + +### 3. Running the App + +You can launch the Streamlit application using the `intugle-streamlit` command or `uvx`: + +```bash +intugle-streamlit +# Or using uvx +uvx --from intugle intugle-streamlit +``` + +Open the URL provided in your terminal (usually `http://localhost:8501`) to access the application. + +## ⚙️ How It Works + +The application guides you through a simple, multi-step process, which is tracked in the sidebar: + +1. **Upload Files**: Start by uploading one or more CSV or Excel files. The app will display a summary of the uploaded tables. +2. **Configure LLM**: In the sidebar, choose your LLM provider (OpenAI, Azure, or Gemini) and enter the necessary API keys and configuration details. +3. **Prepare Data**: Review the uploaded tables. You can rename tables and modify columns (rename, or ignore/drop them). Once you are satisfied, click **"Freeze column names"** to lock in your changes. +4. **Build Semantic Model**: After preparing your data, click **"Create Semantic Model"**. You will be prompted to provide a "domain" (e.g., *Healthcare*, *Manufacturing*) to give the LLM context. The app will then profile the data and generate a business glossary for each table. +5. **Predict Links**: Once profiling is complete, click **"Run Link Prediction"** to discover the relationships between your datasets. +6. **Explore & Download**: View the results as an interactive graph or a detailed table. You can download the underlying YAML configuration files from the sidebar at any time. diff --git a/pyproject.toml b/pyproject.toml index 7579649..ece0ad8 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -4,7 +4,7 @@ build-backend = "setuptools.build_meta" [project] name = "intugle" -version = "1.0.11" +version = "1.0.12" authors = [ { name="Intugle", email="hello@intugle.ai" }, ] @@ -69,6 +69,15 @@ postgres = [ "sqlglot>=27.20.0", ] +streamlit = [ + "streamlit==1.50.0", + "pyngrok==7.4.0", + "python-dotenv==1.1.1", + "xlsxwriter==3.2.9", + "plotly", + "graphviz" +] + [project.urls] "Homepage" = "https://github.com/Intugle/data-tools" @@ -76,7 +85,7 @@ postgres = [ [project.scripts] intugle-mcp = "intugle.mcp.server:main" -intugle-streamlit = "intugle.cli:export_data" +intugle-streamlit = "intugle.cli:run_streamlit_app" [dependency-groups] test = [ @@ -111,7 +120,7 @@ src = ["src"] where = ["src"] [tool.setuptools.package-data] -"intugle" = ["**/*.yaml", "**/*.txt", "**/*.pkl", "mcp/semantic_layer/prompts/*.md"] +"intugle" = ["**/*.yaml", "**/*.txt", "**/*.pkl", "mcp/semantic_layer/prompts/*.md", "streamlit_app/**/*"] [tool.pytest.ini_options] markers = [ diff --git a/src/intugle/adapters/factory.py b/src/intugle/adapters/factory.py index 8ad8a2f..77e3881 100644 --- a/src/intugle/adapters/factory.py +++ b/src/intugle/adapters/factory.py @@ -75,7 +75,7 @@ def get_dataset_data_type(cls) -> Type[Any]: return Any if len(cls.config_types) == 1: return cls.config_types[0] - return Union[tuple(cls.config_types)] # type: ignore + return Union[tuple(cls.config_types)] # noqa: UP007 @classmethod def create(cls, df: Any) -> Adapter: diff --git a/src/intugle/cli.py b/src/intugle/cli.py new file mode 100644 index 0000000..ab126a9 --- /dev/null +++ b/src/intugle/cli.py @@ -0,0 +1,45 @@ +import importlib.util +import os +import subprocess + + +def run_streamlit_app(): + # A list of the required packages for the Streamlit app to run. + # These correspond to the dependencies in the `[project.optional-dependencies].streamlit` section of pyproject.toml. + required_modules = { + "streamlit": "streamlit", + "pyngrok": "pyngrok", + "dotenv": "python-dotenv", + "xlsxwriter": "xlsxwriter", + "plotly": "plotly", + "graphviz": "graphviz", + } + + missing_modules = [] + for module_name, package_name in required_modules.items(): + if not importlib.util.find_spec(module_name): + missing_modules.append(package_name) + + if missing_modules: + print("Error: The Streamlit app is missing required dependencies.") + print("The following packages are not installed:", ", ".join(missing_modules)) + print("\nTo use the Streamlit app, please install 'intugle' with the 'streamlit' extra:") + print(" pip install 'intugle[streamlit]'") + return + + # Get the absolute path to the main.py of the Streamlit app + app_dir = os.path.join(os.path.dirname(__file__), 'streamlit_app') + app_path = os.path.join(app_dir, 'main.py') + + # Ensure the app_path exists + if not os.path.exists(app_path): + print(f"Error: Streamlit app not found at {app_path}") + return + + # Run the Streamlit app using subprocess, setting the working directory + print(f"Launching Streamlit app from: {app_path} with working directory {app_dir}") + subprocess.run(["streamlit", "run", app_path], cwd=app_dir) + + +if __name__ == "__main__": + run_streamlit_app() diff --git a/streamlit_app/.streamlit/config.toml b/src/intugle/streamlit_app/.streamlit/config.toml similarity index 100% rename from streamlit_app/.streamlit/config.toml rename to src/intugle/streamlit_app/.streamlit/config.toml diff --git a/streamlit_app/README.md b/src/intugle/streamlit_app/README.md similarity index 99% rename from streamlit_app/README.md rename to src/intugle/streamlit_app/README.md index aca8432..bd1fa08 100644 --- a/streamlit_app/README.md +++ b/src/intugle/streamlit_app/README.md @@ -1,4 +1,4 @@ -# Intugle - Streamlit App +w# Intugle - Streamlit App This Streamlit application provides an interactive web interface for the `intugle` library. It allows users to upload their tabular data (CSV/Excel), configure a Large Language Model (LLM), and step through the process of building a semantic data model. The app profiles the data, generates a business glossary, identifies relationships between datasets, and visualizes the resulting semantic graph. diff --git a/streamlit_app/helper.py b/src/intugle/streamlit_app/helper.py similarity index 99% rename from streamlit_app/helper.py rename to src/intugle/streamlit_app/helper.py index e48d29e..09ffc1a 100644 --- a/streamlit_app/helper.py +++ b/src/intugle/streamlit_app/helper.py @@ -175,6 +175,7 @@ def safe_filename(name: str, ext: str) -> str: str A sanitized filename like 'my_table.csv'. """ + name = os.path.basename(name) # Sanitize against path traversal base = re.sub(r"[^A-Za-z0-9_.-]+", "_", name).strip("._") if not base: base = "table" @@ -904,7 +905,8 @@ def plotly_table_graph( node_x, node_y, node_text, node_deg, node_labels = [], [], [], [], [] for n in G.nodes(): x, y = pos[n] - node_x.append(x); node_y.append(y) + node_x.append(x) + node_y.append(y) indeg = G.in_degree(n) outdeg = G.out_degree(n) node_text.append(f"{n}
in: {indeg} • out: {outdeg}") @@ -947,14 +949,16 @@ def edge_hover(u: str, v: str, data: Mapping[str, Any]) -> str: # Fast path: one trace with constant width (keeps things interactive for big graphs) edge_x, edge_y, edge_hover_texts = [], [], [] for u, v, data in edges_list: - x0, y0 = pos[u]; x1, y1 = pos[v] + x0, y0 = pos[u] + x1, y1 = pos[v] edge_x += [x0, x1, None] edge_y += [y0, y1, None] edge_hover_texts.append(edge_hover(u, v, data)) # midpoint label text mx, my = (x0 + x1) / 2, (y0 + y1) / 2 - edge_label_x.append(mx); edge_label_y.append(my) + edge_label_x.append(mx) + edge_label_y.append(my) if len(data["labels"]) == 1: edge_label_text.append(data["labels"][0]) else: @@ -974,7 +978,8 @@ def edge_hover(u: str, v: str, data: Mapping[str, Any]) -> str: else: # Accurate path: one trace per edge so we can vary width by mean accuracy for u, v, data in edges_list: - x0, y0 = pos[u]; x1, y1 = pos[v] + x0, y0 = pos[u] + x1, y1 = pos[v] acc_mean = sum(data["accs"]) / max(1, len(data["accs"])) width = max(edge_min_width, acc_mean * edge_width_scale) @@ -992,7 +997,8 @@ def edge_hover(u: str, v: str, data: Mapping[str, Any]) -> str: # midpoint label mx, my = (x0 + x1) / 2, (y0 + y1) / 2 - edge_label_x.append(mx); edge_label_y.append(my) + edge_label_x.append(mx) + edge_label_y.append(my) if len(data["labels"]) == 1: edge_label_text.append(data["labels"][0]) else: diff --git a/streamlit_app/intugle_assets/Intugle_main_logo.png b/src/intugle/streamlit_app/intugle_assets/Intugle_main_logo.png similarity index 100% rename from streamlit_app/intugle_assets/Intugle_main_logo.png rename to src/intugle/streamlit_app/intugle_assets/Intugle_main_logo.png diff --git a/src/intugle/streamlit_app/main.py b/src/intugle/streamlit_app/main.py new file mode 100644 index 0000000..8efccf6 --- /dev/null +++ b/src/intugle/streamlit_app/main.py @@ -0,0 +1,1294 @@ +# ----------------------- +# Import Packages +# ----------------------- +# Standard library +import hashlib +import os +import re +import shutil +import time + +from datetime import datetime +from pathlib import Path +from typing import Dict, Literal + +# Third-party +import pandas as pd +import streamlit as st + +from graphviz import Digraph +from langchain_google_genai import ChatGoogleGenerativeAI +from langchain_openai import AzureChatOpenAI, ChatOpenAI + +from intugle.streamlit_app.helper import ( + _csv_path_for, + _normalized_table_name, + _on_name_edit, + _on_select_change, + add_table_to_state, + build_yaml_zip, + clean_table_name, + clear_cache, + ensure_unique_name, + get_secret, + go_semantic_all, + llm_ready_check, + normalize_col_name, + read_bytes_to_df, + rename_table_in_state, + safe_filename, + show_links_graph_and_table, + sizeof_mb, + standardize_columns, + validate_column_names, +) + +# Local package: intugle +try: + from intugle import SemanticModel + from intugle.analysis.models import DataSet + from intugle.core.settings import settings +except Exception: + st.error("`intugle` package not available. Please install and restart the app.") + st.stop() + +# ----------------------- +# Config & constants +# ----------------------- +# MAX_MB = 25 +# ACCEPTED_TYPES = (".csv", ".xls", ".xlsx") +# PREVIEW_ROWS = 100 + +# st.set_page_config( +# page_title="Intugle • Semantic Model Builder", +# page_icon="🟣", +# layout="wide" +# ) +# def set_base_dir_once(): +# """ +# Create and store a per-session base directory exactly once. +# Default folder name is current datetime, e.g., 2025-10-19_14-32-07. +# Files go under //... +# """ +# if "BASE_DIR" in st.session_state: +# return + +# # format timestamp (safe for folders) +# now = datetime.now() +# stamp = now.strftime("%Y-%m-%d_%H-%M-%S") + +# base = str(stamp) +# st.session_state.BASE_DIR = base +# st.toast("Base dir:"+ st.session_state.BASE_DIR) + + +# def relpath(path: str) -> Path: +# """Build paths under the session's base dir.""" +# base = Path(st.session_state["BASE_DIR"]) +# return base / path # returns a Path + + +# # --- call once at the top of your script --- +# set_base_dir_once() +# # Folders for I/O +# # INPUT_DIR = Path("input") +# INPUT_DIR = relpath("input") +# # MODIFIED_DIR = Path("modified_input") +# MODIFIED_DIR = relpath("modified_input") +# INPUT_DIR.mkdir(parents=True, exist_ok=True) +# MODIFIED_DIR.mkdir(parents=True, exist_ok=True) +# ASSET_DIR = relpath("models") # change if your folder is named differently +# settings.PROJECT_BASE = str(ASSET_DIR) +# ICON_DIR = relpath("intugle_assets") +# # make directory if dosent exist +# if not ASSET_DIR.exists(): +# ASSET_DIR.mkdir(parents=True, exist_ok=True) +# if not ICON_DIR.exists(): +# ICON_DIR.mkdir(parents=True, exist_ok=True) + + +# ----------------------- +# Config & constants +# ----------------------- + +# --- Page config MUST be first in the script --- +st.set_page_config( + page_title="Intugle • Semantic Model Builder", + page_icon="🟣", + layout="wide", +) +# ---- Constants ---- +MAX_MB: int = 25 +ACCEPTED_TYPES = (".csv", ".xls", ".xlsx") +PREVIEW_ROWS: int = 100 + + +# ---- Session-scoped base directory helpers ---- +def _timestamp_folder() -> str: + """Return a filesystem-safe timestamp like '2025-10-19_14-32-07'.""" + return datetime.now().strftime("%Y-%m-%d_%H-%M-%S") + + +def ensure_base_dir() -> Path: + """ + Create and store a per-session base directory exactly once. + Returns the Path to the base dir. + """ + created = False + if "BASE_DIR" not in st.session_state: + st.session_state["BASE_DIR"] = _timestamp_folder() + created = True + + base = Path(st.session_state["BASE_DIR"]) + if created: + st.toast(f"Base dir: {base}") + return base + + +def relpath(path: str | Path) -> Path: + """Build a path under the session's base dir.""" + base = ensure_base_dir() + return base / Path(path) + + +# ---- Initialize I/O directories ---- +def init_io_dirs() -> Dict[str, Path]: + """ + Prepare the working directories for the session and return them. + """ + paths = { + "INPUT_DIR": relpath("input"), + "MODIFIED_DIR": relpath("modified_input"), + "ASSET_DIR": relpath("models"), # adjust if your folder name differs + # "ICON_DIR": relpath("intugle_assets"), + } + + for p in paths.values(): + p.mkdir(parents=True, exist_ok=True) + + # If intugle settings is available, set project base + try: + # 'settings' should be imported earlier from intugle.core.settings + settings.PROJECT_BASE = str(paths["ASSET_DIR"]) + except NameError: + # settings not imported/available; skip without failing the app + pass + + return paths + + +# ---- Call once near top of script ---- +dirs = init_io_dirs() +INPUT_DIR = dirs["INPUT_DIR"] +MODIFIED_DIR = dirs["MODIFIED_DIR"] +ASSET_DIR = dirs["ASSET_DIR"] +# ICON_DIR = dirs["ICON_DIR"] +# ----------------------- +# Session state initialization & lightweight router +# ----------------------- + +# Types (optional, for clarity in editors) +Route = Literal["home", "semantic"] + +DEFAULT_STATE = { + # Core app state + "uploader_key": 0, + "tables": {}, + "ingest_log": [], + "seen_files": {}, + "main_modify_select": "none", + "llm_choice": "openai", + "llm_config": {}, + # Router + semantic flow + "route": "home", # type: Route + "semantic_table": None, + "data_profiling_glossary": False, + "semantic_model_done": False, + "creds_saved": False, + "links_list": None, + # One-shot flags + "just_uploaded_rerun": False, +} + + +def init_session_state(defaults: dict) -> None: + """Idempotently seed st.session_state with defaults.""" + for k, v in defaults.items(): + st.session_state.setdefault(k, v) + + +# Initialize once at top of file +init_session_state(DEFAULT_STATE) + +# Clear one-shot flags if set +_ = st.session_state.pop("just_uploaded_rerun", False) + +# Convenience: whether to show the upload expander +upload_expander: bool = st.session_state["uploader_key"] == 0 + +# --- init state (add once) --- + +# ---- sensible defaults so it doesn't crash on first run +# st.session_state.setdefault("seen_files", {}) +# st.session_state.setdefault("creds_saved", False) +# st.session_state.setdefault("route", "home") +# st.session_state.setdefault("data_profiling_glossary", False) +# st.session_state.setdefault("semantic_model_done", False) + +# # ----- Session init ----- +# if "creds_saved" not in st.session_state: +# st.session_state.creds_saved = False + +# # Provider choice: "openai", "azure-openai", "gemini" +# if "llm_choice" not in st.session_state: +# st.session_state.llm_choice = "openai" + +# # Store normalized config for the active provider +# if "llm_config" not in st.session_state: +# st.session_state.llm_config = {} + + +# # Initialize state +# # if "uploader_key" not in st.session_state: st.session_state["uploader_key"] = 0 +# if "tables" not in st.session_state: st.session_state["tables"] = {} +# if "ingest_log" not in st.session_state: st.session_state["ingest_log"] = [] +# if "seen_files" not in st.session_state: st.session_state["seen_files"] = {} +# if "main_modify_select" not in st.session_state: st.session_state["main_modify_select"] = 'none' + +# # --- Simple router state --- +# if "route" not in st.session_state: +# st.session_state["route"] = "home" # "home" | "semantic" +# if "semantic_table" not in st.session_state: +# st.session_state["semantic_table"] = None +# if "data_profiling_glossary" not in st.session_state: +# st.session_state["data_profiling_glossary"] = False +# if "semantic_model_done" not in st.session_state: +# st.session_state["semantic_model_done"] = False + + +# # ---- init once (top of file) ---- +# st.session_state.setdefault("uploader_key", 0) +# st.session_state.setdefault("just_uploaded_rerun", False) + +# # If we just forced a rerun, clear the flag (one-shot) +# if st.session_state.pop("just_uploaded_rerun", False): +# pass + + +# if st.session_state["uploader_key"] ==0: +# upload_expander = True +# else: +# upload_expander = False + + +# ----------------------- +# UI: Header +# ----------------------- + +# download image +# if upload_expander: +# # url = "https://commons.wikimedia.org/wiki/File:Intugle_icon.png" +# url = "https://commons.wikimedia.org/wiki/File:Intugle_main_logo.png" +# path = download_image(url, save_folder=ICON_DIR) +# print(f"Image downloaded to: {path}") + +# Layout: logo + title side by side +_, tcol1, tcol2 = st.columns([1, 2, 0.5]) + +with tcol1: + st.title(":violet[Intugle Semantic Modeler]") + +with tcol2: + # st.link_button("🌐 Website", "https://intugle.ai/",type="tertiary") + st.link_button("GitHub Repo", "https://intugle.github.io/data-tools/") + + +logo_path = os.path.join(os.path.dirname(__file__), "intugle_assets", "Intugle_main_logo.png") +st.logo( + # 'https://commons.wikimedia.org/wiki/File:Intugle_icon.png', + logo_path, + size="large", +) + + +# ========================= +# Status Bar (single row) +# ========================= + +# ---- your booleans +upload_done = bool(st.session_state["seen_files"]) +creds_done = bool(st.session_state["creds_saved"]) +freeze_done = st.session_state.get("route") == "semantic_all" +profiling_done = bool(st.session_state["data_profiling_glossary"]) +links_done = bool(st.session_state["semantic_model_done"]) + +# ---- stages in order +stages = [ + ("Upload Files", upload_done), + ("LLM Credentials", creds_done), + ("Rename Tables & Columns", freeze_done), + ("Profiling & Glossary", profiling_done), + ("Semantic Link Identification", links_done), +] + +# ---- render in sidebar +with st.sidebar: + st.markdown("## Progress") + + st.markdown( + """ +
+""", + unsafe_allow_html=True, + ) + + html_parts = [] + for label, done in stages: + icon = "✔️" if done else "🟣" + state_cls = "done" if done else "todo" + html_parts.append( + f'
{icon}{label}
' + ) + + st.markdown("".join(html_parts) + "
", unsafe_allow_html=True) + st.divider() +# ------------------------------------------ +# st.title("Intugle Semantic Modeler") + + +if st.session_state["route"] == "home": + st.subheader("How it works") + + dot = Digraph( + name="intugle_flow", + graph_attr={ + "rankdir": "LR", # left → right + "bgcolor": "white", + "fontsize": "10", + "pad": "0.2", + }, + node_attr={ + "shape": "rect", + "style": "rounded,filled", + "fontname": "Inter, Helvetica, Arial, sans-serif", + "color": "#8B5CF6", # Intugle purple border + "fillcolor": "#F3E8FF", # soft lilac fill + "fontsize": "11", + }, + edge_attr={ + "color": "#8B5CF6", + "fontname": "Inter, Helvetica, Arial, sans-serif", + "fontsize": "10", + }, + ) + + # Nodes + dot.node("U", "Upload CSV/Excel\n(one or many)") + dot.node("S", "Size ≤ 25 MB?", shape="diamond", fillcolor="#EDE9FE") + dot.node("P", "Profile columns\n• Uniqueness\n• Completeness\n• Data type") + dot.node("G", "Generate glossary\nfor all columns") + dot.node("M", "Build semantic model\n(Currently single link per table pair)") + dot.node("X", "Skip file\n(too large)", fillcolor="#FFE4E6", color="#EF4444") + + # Edges + dot.edge("U", "S") + dot.edge("S", "P", label="Yes") + dot.edge("S", "X", label="No") + dot.edge("P", "G") + dot.edge("G", "M") + + # Render in Streamlit + st.graphviz_chart(dot, width="stretch") + + st.subheader("1) Upload Files") + with st.expander("Upload Your Files here..", expanded=upload_expander): + uploaded_files = st.file_uploader( + "Drop files here or browse", + type=[ext.lstrip(".") for ext in ACCEPTED_TYPES], + accept_multiple_files=True, + key=f"uploader_{st.session_state['uploader_key']}", + ) + + if uploaded_files: + st.caption(f"Received {len(uploaded_files)} file(s).") + + for upl in uploaded_files: + # Read bytes once so we can hash and parse from the same buffer + file_bytes = upl.getvalue() + size_mb = sizeof_mb(len(file_bytes)) + sha = hashlib.sha1(file_bytes).hexdigest() + + # Skip files already seen by content hash (prevents duplicates on reruns) + if sha in st.session_state["seen_files"]: + continue + + # Build a log row; we only append if we actually process/skip/error + log_row = { + "source_file": upl.name, + "size_mb": size_mb, + "status": "⏳ pending", + "table_name": None, + "rows": None, + "cols": None, + "details": "", + } + + # Enforce size + if size_mb > MAX_MB: + st.warning(f"Skipping **{upl.name}** because it exceeds {MAX_MB} MB.") + # log_row["status"] = "⛔ too large (>25MB)" + # log_row["details"] = "Skipped due to size limit." + # st.session_state["ingest_log"].append(log_row) + continue + + # Parse + try: + with st.spinner(f"Reading `{upl.name}`..."): + df, note = read_bytes_to_df(upl.name, file_bytes) + except Exception as e: + st.error(f"Failed to read **{upl.name}**: {e}") + # log_row["status"] = "❌ read error" + # log_row["details"] = str(e) + # st.session_state["ingest_log"].append(log_row) + continue + + # Standardize columns + with st.spinner(f"Standardizing columns for `{upl.name}`..."): + df = standardize_columns(df) + + # # Initial table name (no extra counters unless a different file collides) + input_path = INPUT_DIR / safe_filename(clean_table_name(upl.name), ".csv") + try: + if (not input_path.exists()) or (hashlib.sha1(input_path.read_bytes()).hexdigest() != sha): + input_path.write_bytes(file_bytes) + except Exception as e: + st.warning(f"Could not save original file for **{upl.name}**: {e}") + + # b) Standardized table -> modified_input/ as CSV (ALWAYS overwrite) + default_name = clean_table_name(upl.name) + unique_default = ensure_unique_name(default_name, st.session_state["tables"]) + + modified_path = MODIFIED_DIR / safe_filename(unique_default, ".csv") + try: + df.to_csv(modified_path, index=False) + # cleanup any legacy xlsx with the same stem + legacy_xlsx = MODIFIED_DIR / safe_filename(unique_default, ".xlsx") + if legacy_xlsx.exists(): + legacy_xlsx.unlink(missing_ok=True) + except Exception as e: + st.error(f"Failed to save standardized CSV to modified_input/: {e}") + continue + # ----------------------------------------------- + + # ----------------------------------------------- + # Metadata + rows_i, cols_i = int(df.shape[0]), int(df.shape[1]) + meta = { + "source_file": upl.name, + "size_mb": size_mb, + "note": note, + "rows": rows_i, + "cols": cols_i, + "timestamp": time.time(), + "modified_path": str(modified_path), # <— new + } + + # Save table and fingerprint + with st.spinner(f"Saving `{unique_default}`..."): + add_table_to_state(unique_default, df, meta) + + st.session_state["seen_files"][sha] = { + "table_name": unique_default, + "source_file": upl.name, + "size_mb": size_mb, + } + + # Log success + log_row.update({ + "status": "✅ loaded", + "table_name": unique_default, + "rows": rows_i, + "cols": cols_i, + "details": note or "", + }) + st.session_state["ingest_log"].append(log_row) + + st.toast("Processing completed for all eligible files.") + # Clear the uploader selection and force exactly one rerun + st.session_state["uploader_key"] += 1 # resets file_uploader + st.session_state["just_uploaded_rerun"] = True # one-shot marker + st.rerun() + + # ----------------------- + # Tables overview (read-only summary) + # ----------------------- + # Normalize ingest_log names from seen_files (ensures latest rename is shown) + for d, info in st.session_state.get("seen_files", {}).items(): + latest_name = info.get("table_name") + for row in st.session_state.get("ingest_log", []): + if row.get("source_file") == info.get("source_file") and row.get("status", "").startswith("✅"): + row["table_name"] = latest_name + + if st.session_state.get("tables") or st.session_state.get("ingest_log"): + with st.expander("1) Review Tables", expanded=False): + # Summary now shows status/details only; no inline actions + if st.session_state.get("ingest_log"): + summary_df = pd.DataFrame(st.session_state["ingest_log"]).copy() + cols_order = ["table_name", "source_file", "size_mb", "rows", "cols", "status", "details"] + summary_df = summary_df[[c for c in cols_order if c in summary_df.columns]] + else: + # Fallback if ingest_log not present + rows = [] + for tname, payload in st.session_state["tables"].items(): + m = payload["meta"] + rows.append({ + "table_name": tname, + "source_file": m["source_file"], + "size_mb": m["size_mb"], + "rows": m["rows"], + "cols": m["cols"], + "status": "✅ loaded", + "details": m.get("note", ""), + }) + summary_df = pd.DataFrame(rows).sort_values("table_name") + + st.dataframe(summary_df, width="stretch", hide_index=True) + else: + # st.info("No tables loaded yet. Upload some CSV/Excel files above to get started!") + + st.toast( + """ + **Welcome to Intugle!** + We’re thrilled to have you here. + """, + icon="🕸️", + ) + + # ----------------------- + # Modify table & columns (MAIN PAGE, under the review table) + # ----------------------- + + # --- Always-visible table picker + synced final name --- + + if not st.session_state.get("tables"): + # st.info("Load tables to modify them here.") + time.sleep(1) + st.toast("Let us Start by uploading files and submit your LLM credentials") + + else: + st.divider() + st.subheader("2) Modify table & columns") + # Ensure state keys exist + st.session_state.setdefault("final_name_user_edited", False) + + # left, right = st.columns([0.5, 0.5]) + + # Build options; always render the selectbox + has_tables = bool(st.session_state.get("tables")) + tnames = sorted(st.session_state["tables"].keys()) if has_tables else [] + options = tnames if has_tables else ["— no tables loaded —"] + col1, col2, col3, col4 = st.columns([2, 0.5, 2, 2]) + with col1: + sel = st.selectbox( + "Select table to modify", + options=options, + index=0, + # key="main_modify_select", + on_change=_on_select_change, + disabled=not has_tables, + help="Choose a loaded table. This is always shown; it will be disabled until you upload data.", + ) + out_fmt = "CSV" + # Or, if you want to show a control but keep a default, uncomment: + # out_fmt = st.radio("Save format", ["CSV", "Excel"], horizontal=True, key="main_modify_fmt") + + # Guard when there are no tables yet + if not has_tables: + st.info("Load tables to modify them here.") + st.stop() + + # Use the selected table + payload = st.session_state["tables"][sel] + df_current = payload["df"] + orig_cols = list(df_current.columns) + with col2: + # add a right emoji arrow + # st.markdown("
➡️
", unsafe_allow_html=True) + st.image( + "https://upload.wikimedia.org/wikipedia/commons/c/ca/Eo_circle_purple_arrow-right.svg", + width=80, # Manually Adjust the width of the image as per requirement + ) + + with col3: + # Final name input that mirrors selection until the user edits it + # st.markdown("**Rename table (optional)**") + final_name_raw = st.text_input( + "Rename table (optional)", + # key="main_modify_final_name", + # value=st.session_state.get("main_modify_final_name", sel) or sel, + value=sel, + on_change=_on_name_edit, + help="Defaults to the selected table name. Will be normalized to lowercase with underscores.", + ) + + # Current paths + current_name = sel + current_path = _csv_path_for(current_name, MODIFIED_DIR) + + # ------------------------------- + # ACTIONS + # ------------------------------- + ( + __, + _, + act_col1, + ___, + ) = st.columns([2, 0.5, 2, 2]) + + # 1) RENAME TABLE (and CSV file) — no column changes here + with act_col1: + if st.button("Rename table", type="primary", width="content", key="btn_rename_table"): + new_name = _normalized_table_name(final_name_raw) + if not new_name: + st.error("Final table name cannot be empty after normalization.") + st.stop() + + if new_name == current_name: + st.toast("Table name unchanged.") + else: + new_path = _csv_path_for(new_name, MODIFIED_DIR) + + # Move/rename the CSV on disk (if it exists). If it doesn't, create from in-memory df. + try: + if current_path.exists(): + # Avoid cross-device rename oddities + current_path.replace(new_path) + else: + # If the file isn't there yet, write the current DataFrame + st.session_state["tables"][current_name]["df"].to_csv(new_path, index=False) + except Exception as e: + st.error(f"Failed to rename CSV file: {e}") + st.stop() + + # Cleanup any legacy .xlsx with the old name + legacy_xlsx = MODIFIED_DIR / safe_filename(current_name, ".xlsx") + legacy_xlsx.unlink(missing_ok=True) + + # Update in-memory state & logs + rename_table_in_state(current_name, new_name) + st.session_state["tables"][new_name]["meta"]["modified_path"] = str(new_path) + for row in st.session_state.get("ingest_log", []): + if row.get("table_name") == current_name: + row["table_name"] = new_name + for _, info in st.session_state.get("seen_files", {}).items(): + if info.get("table_name") == current_name: + info["table_name"] = new_name + + # Sync the selectbox and input defaults + st.session_state["main_modify_select"] = new_name + st.session_state["main_modify_final_name"] = new_name + st.session_state["final_name_user_edited"] = False + + st.success(f"Renamed **{current_name}** → **{new_name}** and updated CSV file.") + st.rerun() + + # Column editor (unchanged) + st.markdown("**Edit or ignore columns**") + editor_src = { + "original_column_name": orig_cols, + "new_column_name": orig_cols.copy(), # default to current names + "ignore_column": [False] * len(orig_cols), + } + edit_df = st.data_editor( + pd.DataFrame(editor_src), + hide_index=True, + width="stretch", + column_config={ + "original_column_name": st.column_config.TextColumn("Original_Column_Name", disabled=True), + "new_column_name": st.column_config.TextColumn( + "New_Column_Name (Editable)", help="Enter desired column names" + ), + "ignore_column": st.column_config.CheckboxColumn( + "Select_column_to_Ignore", help="Drop this column for sematic model" + ), + }, + key="main_modify_editor", + ) + # ---------- Editor UI already created above ---------- + # sel, df_current, orig_cols, final_name_raw, edit_df exist + + # 2) FREEZE COLUMN NAMES — apply edits/ignores & save back to SAME file name (CSV overwrite) + # with act_col2: + if st.button("Freeze column names", type="primary", width="content", key="btn_freeze_cols"): + # Pull edits + new_names_input = list(edit_df["new_column_name"]) + ignored_flags = list(edit_df["ignore_column"]) + + # Validate proposed names + errs = validate_column_names(orig_cols, new_names_input, ignored_flags) + if errs: + for e in errs: + st.error(e) + st.stop() + + # Build modified DataFrame + keep_idx = [i for i, ig in enumerate(ignored_flags) if not ig] + kept_cols = [orig_cols[i] for i in keep_idx] + new_names = [normalize_col_name(new_names_input[i]) for i in keep_idx] + + df_mod = df_current.loc[:, kept_cols].copy() + df_mod.columns = new_names + + # Determine the *current* table name (which might have been renamed already) + # live_name = st.session_state["main_modify_select"] + live_name = final_name_raw + st.info(live_name) + live_path = _csv_path_for(live_name, MODIFIED_DIR) + + # Save back to the SAME file (overwrite) + try: + df_mod.to_csv(live_path, index=False) + except Exception as e: + st.error(f"Failed to write CSV: {e}") + st.stop() + time.sleep(3) + # Update in-memory table payload + st.session_state["tables"][live_name]["df"] = df_mod + st.session_state["tables"][live_name]["meta"]["rows"] = int(df_mod.shape[0]) + st.session_state["tables"][live_name]["meta"]["cols"] = int(df_mod.shape[1]) + st.session_state["tables"][live_name]["meta"]["modified_path"] = str(live_path) + + st.success(f"Column names frozen and saved to `{live_path.name}`.") + st.rerun() + + +# ----------------------- +# Sidebar +# ----------------------- +with st.sidebar: + st.header("🔑 LLM Settings") + + # If already saved this session: show summary + "Change settings" + if st.session_state.creds_saved: + # st.write(f"**LLM Service:** `{provider_display_string()}`") + + choice = st.session_state.llm_choice + cfg = st.session_state.llm_config + + if choice == "openai": + # st.write(f"**OpenAI Model Loaded** `{cfg.get('model','')}`") + # st.write(f"**Model:** `{cfg.get('model','')}`") + # st.write(f"**OPENAI_API_KEY:** {mask(get_secret('OPENAI_API_KEY'))}") + # pip install -U langchain-openai + + llm_temp = ChatOpenAI( + model=os.environ["LLM_PROVIDER"], # pick your OpenAI model + temperature=0, + # api_key="...", # or set env var: OPENAI_API_KEY=... + # base_url="...", # only if you're using a compatible proxy + ) + + messages = [ + ( + "system", + "You are a helpful assistant that translates French to English. Translate the user sentence.", + ), + ("human", "Le point de terminaison OpenAI a été chargé avec succès."), + ] + + ai_msg = llm_temp.invoke(messages) + st.success(ai_msg.content) + + # st.success(f"**OpenAI API Key load successful:**") + + elif choice == "azure-openai": + # Import Azure OpenAI + model = os.environ.get("LLM_PROVIDER", "").split(":")[-1] + llm_temp = AzureChatOpenAI( + azure_deployment=model, # your deployed model name or alias + # azure_endpoint="https://.openai.azure.com/", + # api_key="...", + # api_version="2024-xx-xx" + ) + messages = [ + ( + "system", + "You are a helpful assistant that translates French to English. Translate the user sentence.", + ), + ("human", "Le point de terminaison Azure OpenAI a été chargé avec succès."), + ] + ai_msg = llm_temp.invoke(messages) + ai_msg = ai_msg.content + # st.success(f"**Azure OpenAI Endpoint load successful:**") + st.success(ai_msg) + # st.write(f'LLM_PROVIDER: {os.environ["LLM_PROVIDER"]}') + # st.write(f'AZURE_OPENAI_API_KEY: {os.environ["AZURE_OPENAI_API_KEY"]}') + # st.write(f'OPENAI_API_VERSION: {os.environ["OPENAI_API_VERSION"]}') + # st.write(f'AZURE_OPENAI_ENDPOINT: {os.environ["AZURE_OPENAI_ENDPOINT"]}') + + elif choice == "gemini": + # st.write(f"**Model:** `{cfg.get('model','')}`") + # Common env names for Gemini: GEMINI_API_KEY or GOOGLE_API_KEY + # st.write(f"**GEMINI/GOOGLE_API_KEY:** {mask(get_secret('GEMINI_API_KEY') or get_secret('GOOGLE_API_KEY'))}") + + llm_temp = ChatGoogleGenerativeAI( + model=os.environ["LLM_PROVIDER"], + temperature=0, + # google_api_key="...", # or set env var: GOOGLE_API_KEY=... + ) + + messages = [ + ( + "system", + "You are a helpful assistant that translates French to English. Translate the user sentence.", + ), + ("human", "Le point de terminaison Google Gemini a été chargé avec succès."), + ] + + ai_msg = llm_temp.invoke(messages) + ai_msg = ai_msg.content + st.success(ai_msg) + # st.success(f"**Gemini API Key load successful:**") + # if st.button("Change settings"): + # st.session_state.creds_saved = False + # st.stop() + + # Otherwise: render the provider picker + provider-specific fields + with st.expander("LLM Settings", expanded=not st.session_state.creds_saved): + provider = st.selectbox( + "Choose LLM provider", + options=["openai", "azure-openai", "google_genai"], + index=["openai", "azure-openai", "google_genai"].index(st.session_state.llm_choice), + help="Pick your LLM backend.", + ) + st.session_state.llm_choice = provider + + # --------- OPENAI --------- + if provider == "openai": + # Pre-fill from env/secrets if present + pre_key = get_secret("OPENAI_API_KEY", "") + model = st.selectbox( + "Model", + # ["openai:gpt-3.5-turbo"], + ["gpt-4o-mini", "gpt-4o", "gpt-3.5-turbo"], + index=0, + ) + api_key = st.text_input( + "OpenAI API key", type="password", value=pre_key, placeholder="sk-********************************" + ) + + if st.button("Save provider settings", type="primary", width="stretch"): + if not api_key or len(api_key) < 20: + st.error("Please enter a valid OpenAI API key.") + else: + st.session_state.llm_config = { + "model": model, + } + # Normalize env vars (and a generic LLM_PROVIDER string if you use it elsewhere) + # save_env({ + # "OPENAI_API_KEY": api_key.strip(), + # "LLM_PROVIDER": f"openai:{model}", + # }) + settings.LLM_PROVIDER = f"openai:{model}" + os.environ["OPENAI_API_KEY"] = api_key.strip() + os.environ["LLM_PROVIDER"] = model + st.session_state.creds_saved = True + st.success("OpenAI settings saved.") + st.rerun() + + # --------- AZURE OPENAI --------- + elif provider == "azure-openai": + pre_key = get_secret("AZURE_OPENAI_API_KEY", "") + pre_endpoint = get_secret("AZURE_OPENAI_ENDPOINT", "https://YOUR-RESOURCE-NAME.openai.azure.com/") + pre_llm_provider = get_secret("LLM_PROVIDER", "") + pre_version = get_secret("OPENAI_API_VERSION", "2024-06-01") + + endpoint = st.text_input( + "Azure OpenAI endpoint", value=pre_endpoint, placeholder="https://.openai.azure.com/" + ) + llm_provider = st.text_input("Provider name", value=pre_llm_provider, placeholder="azure_openai:gpt-4o") + api_version = st.text_input("API version", value=pre_version, placeholder="2024-06-01") + api_key = st.text_input( + "Azure OpenAI API key", type="password", value=pre_key, placeholder="****************" + ) + + if st.button("Save provider settings", type="primary", width="stretch"): + errs = [] + if not endpoint.startswith("http"): + errs.append("Endpoint must start with http/https.") + if not llm_provider: + errs.append("LLM provider name is required.") + if not api_version: + errs.append("API version is required.") + if not api_key or len(api_key) < 20: + errs.append("Enter a valid Azure OpenAI API key.") + if errs: + for e in errs: + st.error(e) + else: + st.session_state.llm_config = { + "endpoint": endpoint.strip(), + "llm_provider": llm_provider.strip(), + "api_version": api_version.strip(), + } + # save_env({ + # "AZURE_OPENAI_ENDPOINT": endpoint.strip(), + # "LLM_PROVIDER": llm_provider.strip(), + # "OPENAI_API_VERSION": api_version.strip(), + # "AZURE_OPENAI_API_KEY": api_key.strip(), + # # "LLM_PROVIDER": f"azure-openai:{deployment.strip()}@{api_version.strip()}", + # }) + os.environ["LLM_PROVIDER"] = str(llm_provider.strip()) + settings.LLM_PROVIDER = str(llm_provider.strip()) + os.environ["AZURE_OPENAI_API_KEY"] = str(api_key.strip()) + os.environ["OPENAI_API_VERSION"] = str(api_version.strip()) + os.environ["AZURE_OPENAI_ENDPOINT"] = str(endpoint.strip()) + + st.session_state.creds_saved = True + st.success("Azure OpenAI settings saved.") + st.rerun() + + # --------- GEMINI --------- + elif provider == "google_genai": + # Gemini keys are commonly under GEMINI_API_KEY or GOOGLE_API_KEY + pre_key = get_secret("GOOGLE_API_KEY") or "" + model = st.selectbox("Model", ["gemini-2.5-pro", "gemini-2.5-flash", "gemini-2.5-flash-lite"], index=0) + api_key = st.text_input( + "Gemini API key", type="password", value=pre_key, placeholder="********************************" + ) + + if st.button("Save provider settings", type="primary", width="stretch"): + if not api_key or len(api_key) < 20: + st.error("Please enter a valid Gemini API key.") + else: + st.session_state.llm_config = {"model": model} + # Set both common env var names so whichever client you use can find it + # save_env({ + # "GEMINI_API_KEY": api_key.strip(), + # "GOOGLE_API_KEY": api_key.strip(), + # "LLM_PROVIDER": f"gemini:{model}", + # }) + os.environ["GOOGLE_API_KEY"] = api_key.strip() + os.environ["LLM_PROVIDER"] = model + settings.LLM_PROVIDER = f"google_genai:{model}" + st.session_state.creds_saved = True + st.success("Gemini settings saved.") + st.rerun() + +with st.sidebar: + # Settings & info + # with st.expander("Reset Settings",expanded=False): + # st.write(f"Max file size per file: **{MAX_MB} MB**") + # st.write(f"Accepted types: {', '.join(ACCEPTED_TYPES)}") + + if st.button("Reset App", type="primary", width="content"): + # st.session_state["tables"] = {} + # st.session_state["ingest_log"] = [] # optional: reset the log + # st.session_state["seen_files"] = {} # important: allow re-uploading same files + # st.session_state["uploader_key"] += 1 # <-- resets the uploader selection + clear_cache() + try: + shutil.rmtree(INPUT_DIR) + shutil.rmtree(ASSET_DIR) + shutil.rmtree(MODIFIED_DIR) + + print("input & modified_input folders found & deleted. You are good to go") + except: # noqa: E722 + print("No folders found. You are good to go") + + st.success("Cleared all loaded tables and upload selection.") + + st.rerun() + + +# ----------------------- ----------------------- ----------------------- ----------------------- ----------------------- +# 5) Create a Semantic Model (uses all files in modified_input/) +# ----------------------- ----------------------- ----------------------- ----------------------- ----------------------- +# ====================== + +# --- tiny helpers (define once) --- + + +def _list_modified_files(): + return sorted(list(MODIFIED_DIR.glob("*.csv")) + list(MODIFIED_DIR.glob("*.xlsx"))) + + +def _source_type_from_suffix(p: Path) -> str: + return "csv" if p.suffix.lower() == ".csv" else "excel" + + +# Find modified files +modified_csv = list(MODIFIED_DIR.glob("*.csv")) +modified_xlsx = list(MODIFIED_DIR.glob("*.xlsx")) +modified_files = modified_csv + modified_xlsx +n_files = len(modified_files) + +# LLM readiness +ok, err = llm_ready_check() +if not ok: + st.error(err) + st.stop() + + +if n_files == 0: + st.info("No tables found. Upload & Freeze at least one table to continue.") +elif st.session_state.get("route") != "semantic_all": + st.caption(f"Found **{n_files}** file(s)") + # Brief preview of filenames (first 5) + show = [p.name for p in modified_files[:5]] + # st.write(", ".join(show) + (" ..." if n_files > 5 else "")) + + # Full-width action button + st.divider() + if st.button( + "🧠 Create Semantic Model from Modified Data", type="primary", width="stretch", key="btn_semantic_all" + ): + go_semantic_all() + + +# ========================= +# Semantic page content +# ========================= +if st.session_state.get("route") == "semantic_all": + # if st.sidebar.button("⬅️ Back to Data", key="btn_semantic_back"): + # st.session_state["route"] = "home" + # st.rerun() + ############################################################################## + with st.sidebar: + # Make the sidebar a full-height flex column + st.markdown( + """ + + """, + unsafe_allow_html=True, + ) + + # --- your existing sidebar content goes here --- + st.markdown("# ") + # ... (status pills, etc.) + + # This flexible spacer expands and pushes what follows to the bottom + st.markdown('
', unsafe_allow_html=True) + + st.divider() + if st.button("⬅️ Back to Data", key="btn_semantic_back", width="stretch"): + st.session_state["route"] = "home" + st.rerun() + ############################################################################## + st.header("3) 🧠 Build Semantic Model") + # 2) Define domain BEFORE using it + st.markdown("#### Domain") + dcol1, dcol2 = st.columns([1, 1]) + with dcol1: + # show whatever is currently saved as the default in the box + domain = st.text_input( + "Enter a domain (e.g., Manufacturing)", + value=st.session_state.get("semantic_domain", "NA"), + key="semantic_domain_input", + ) + + # with dcol2: + if st.button("Set Domain", type="primary", width="content"): + # Only commit when the button is clicked + committed = (st.session_state.get("semantic_domain_input") or "").strip() or "NA" + st.session_state["semantic_domain"] = committed + st.toast(f"Domain set to: **{committed}**") + + # Require domain + + if ( + not domain.strip() + or domain.strip().upper() == "NA" + or domain.strip() == "" + or not bool(re.search(r"[A-Za-z0-9]", domain.strip())) + ): + st.warning("Please enter a valid domain before building the semantic model.") + st.stop() + # 1) Define files BEFORE using them + files = _list_modified_files() + if not files: + st.warning("No files found in modified_input/. Go back and save at least one modified table.") + if st.button("⬅️ Back"): + st.session_state["route"] = "home" + st.rerun() + st.stop() + + st.caption(f"Semantic Model will be built for **{len(files)}** table(s).") + + gdf_iter_ph = st.empty() + # optional: keep a cumulative copy if you still want it elsewhere + st.session_state.setdefault("semantic_gdf_all", None) + + total = len(files) + my_bar = st.progress(0, text="Preparing data for semantic model creation. Please wait.") + + for idx, p in enumerate(files, start=1): # start=2 because first file already processed + if idx == 1: + # Initialize with the first file + first = files[0] + data_sources = {first.stem: {"path": str(first), "type": _source_type_from_suffix(first)}} + + with st.spinner(f"Initializing Semantic Model with `{first.name}` …"): + sm = SemanticModel(data_input=data_sources, domain=domain.strip()) + st.session_state["sm"] = sm + + ds_name = p.stem + + with st.spinner(f"Adding dataset `{ds_name}` …"): + new_ds = DataSet({"path": str(p), "type": _source_type_from_suffix(p)}, name=ds_name) + sm.datasets[ds_name] = new_ds + + with st.spinner(f"Profiling `{ds_name}` ({idx}/{total})…"): + sm.profile() + with st.spinner(f"Generating glossary for `{ds_name}` …"): + sm.generate_glossary() + + # with st.status(f"Preparing data to build semantic model `{ds_name}` …", expanded=True) as status: + # new_ds = DataSet({"path": str(p), "type": _source_type_from_suffix(p)}, name=ds_name) + # sm.datasets[ds_name] = new_ds + # st.write(f"Profiling `{ds_name}` ({idx}/{total}) …") + # sm.profile() + # st.write(f"Generating glossary for `{ds_name}` …") + # sm.generate_glossary() + percent_complete = idx / total + my_bar.progress( + percent_complete, + text=f"Preparing data for semantic model creation {round(percent_complete * 100)}%. Please wait.", + ) + + try: + profiling_df = sm.profiling_df.copy() + glossary_df = sm.glossary_df.copy() + + gdf_this = pd.merge( + glossary_df, + profiling_df, + on=["table_name", "column_name"], + how="left", + suffixes=("_gloss", "_prof"), + ) + + # ---- FLUSH previous view and show ONLY this iteration ---- + gdf_iter_ph.empty() + with gdf_iter_ph.container(): + st.toast(f"##### ✅ Glossary for `{ds_name}` ({idx}/{total})") + # add title to the table + st.markdown(f"### Glossary & Profiling details for `{idx}` tables") + st.dataframe( + gdf_this[ + [ + "table_name", + "column_name", + "datatype_l1", + "datatype_l2", + "count", + "null_count", + "distinct_count", + "uniqueness", + "completeness", + "business_glossary", + "business_tags", + "sample_data", + ] + ], + hide_index=True, + width="stretch", + ) + st.session_state["sm"] = sm + + except Exception as e: + st.warning(f"Could not render glossary view for `{ds_name}`: {e}") + + st.success("Semantic model profiling & glossary generation completed for all tables.") + if not st.session_state["data_profiling_glossary"]: + st.session_state["data_profiling_glossary"] = True + st.rerun() + + # Link prediction and back button (unchanged) + st.divider() + st.subheader("4) Link Prediction") + + if "sm" not in st.session_state: + st.info("Build the semantic model first to enable link prediction.") + else: + sm = st.session_state["sm"] + if st.button("🔗 Run Link Prediction", type="primary", key="btn_link_pred", width="stretch"): + with st.spinner("Predicting links …"): + sm.predict_links() + predictor_instance = sm.link_predictor + st.session_state["links_list"] = predictor_instance.links # <-- SAVE to state + st.session_state["semantic_model_done"] = True # <-- FLAG done + st.success("Link prediction completed.") + + # --- render results if we have them --- + if st.session_state.get("semantic_model_done") and st.session_state.get("links_list"): + try: + show_links_graph_and_table(st.session_state["links_list"]) # <-- use saved list + except Exception as e: + st.warning(f"Could not render link prediction results: {e}") + # if st.button("🔗 Run Link Prediction", type="primary", width="stretch", key="btn_link_pred"): + # with st.spinner("Predicting links …"): + # sm.predict_links() + # # After running the pipeline... + # predictor_instance = sm.link_predictor + + # # Now you can use all the methods of the LinkPredictor + # links_list = predictor_instance.links + # st.success("Link prediction completed.") + + # try: + # # ---------- usage ---------- + # show_links_graph_and_table(links_list) # <-- pass your list[PredictedLink] here + # except Exception as e: + # st.warning(f"Could not render link prediction results: {e}") + + # if st.session_state["semantic_model_done"] == False: + # st.session_state["semantic_model_done"] = True + # st.rerun() + + with st.sidebar: + zip_bytes, file_count = build_yaml_zip(ASSET_DIR) + if file_count: + st.download_button( + label=f"Download {file_count} YAML file{'s' if file_count != 1 else ''} (ZIP)", + data=zip_bytes, + file_name=f"yaml_assets_{time.strftime('%Y%m%d_%H%M%S')}.zip", + mime="application/zip", + width="stretch", + ) + else: + st.info(f"No .yml or .yaml files found in '{ASSET_DIR}/'.") diff --git a/streamlit_app/requirements.txt b/src/intugle/streamlit_app/requirements.txt similarity index 100% rename from streamlit_app/requirements.txt rename to src/intugle/streamlit_app/requirements.txt diff --git a/streamlit_app/main.py b/streamlit_app/main.py deleted file mode 100644 index f3b13ca..0000000 --- a/streamlit_app/main.py +++ /dev/null @@ -1,1229 +0,0 @@ - -# ----------------------- -# Import Packages -# ----------------------- -# Standard library -import hashlib -import os -import re -import shutil -import time - -from datetime import datetime -from pathlib import Path -from typing import Dict, Literal - -# Third-party -import pandas as pd -import streamlit as st - -from graphviz import Digraph -from langchain_google_genai import ChatGoogleGenerativeAI -from langchain_openai import AzureChatOpenAI, ChatOpenAI - -# Local package: intugle -try: - from intugle import SemanticModel - from intugle.analysis.models import DataSet - from intugle.core.settings import settings -except Exception: - st.error("`intugle` package not available. Please install and restart the app.") - st.stop() -from helper import * -from helper import _csv_path_for, _normalized_table_name, _on_name_edit, _on_select_change - -# ----------------------- -# Config & constants -# ----------------------- -# MAX_MB = 25 -# ACCEPTED_TYPES = (".csv", ".xls", ".xlsx") -# PREVIEW_ROWS = 100 - -# st.set_page_config( -# page_title="Intugle • Semantic Model Builder", -# page_icon="🟣", -# layout="wide" -# ) -# def set_base_dir_once(): -# """ -# Create and store a per-session base directory exactly once. -# Default folder name is current datetime, e.g., 2025-10-19_14-32-07. -# Files go under //... -# """ -# if "BASE_DIR" in st.session_state: -# return - -# # format timestamp (safe for folders) -# now = datetime.now() -# stamp = now.strftime("%Y-%m-%d_%H-%M-%S") - -# base = str(stamp) -# st.session_state.BASE_DIR = base -# st.toast("Base dir:"+ st.session_state.BASE_DIR) - - -# def relpath(path: str) -> Path: -# """Build paths under the session's base dir.""" -# base = Path(st.session_state["BASE_DIR"]) -# return base / path # returns a Path - - -# # --- call once at the top of your script --- -# set_base_dir_once() -# # Folders for I/O -# # INPUT_DIR = Path("input") -# INPUT_DIR = relpath("input") -# # MODIFIED_DIR = Path("modified_input") -# MODIFIED_DIR = relpath("modified_input") -# INPUT_DIR.mkdir(parents=True, exist_ok=True) -# MODIFIED_DIR.mkdir(parents=True, exist_ok=True) -# ASSET_DIR = relpath("models") # change if your folder is named differently -# settings.PROJECT_BASE = str(ASSET_DIR) -# ICON_DIR = relpath("intugle_assets") -# # make directory if dosent exist -# if not ASSET_DIR.exists(): -# ASSET_DIR.mkdir(parents=True, exist_ok=True) -# if not ICON_DIR.exists(): -# ICON_DIR.mkdir(parents=True, exist_ok=True) - - -# ----------------------- -# Config & constants -# ----------------------- - -# --- Page config MUST be first in the script --- -st.set_page_config( - page_title="Intugle • Semantic Model Builder", - page_icon="🟣", - layout="wide", -) -# ---- Constants ---- -MAX_MB: int = 25 -ACCEPTED_TYPES = (".csv", ".xls", ".xlsx") -PREVIEW_ROWS: int = 100 - - -# ---- Session-scoped base directory helpers ---- -def _timestamp_folder() -> str: - """Return a filesystem-safe timestamp like '2025-10-19_14-32-07'.""" - return datetime.now().strftime("%Y-%m-%d_%H-%M-%S") - - -def ensure_base_dir() -> Path: - """ - Create and store a per-session base directory exactly once. - Returns the Path to the base dir. - """ - created = False - if "BASE_DIR" not in st.session_state: - st.session_state["BASE_DIR"] = _timestamp_folder() - created = True - - base = Path(st.session_state["BASE_DIR"]) - if created: - st.toast(f"Base dir: {base}") - return base - - -def relpath(path: str | Path) -> Path: - """Build a path under the session's base dir.""" - base = ensure_base_dir() - return base / Path(path) - - -# ---- Initialize I/O directories ---- -def init_io_dirs() -> Dict[str, Path]: - """ - Prepare the working directories for the session and return them. - """ - paths = { - "INPUT_DIR": relpath("input"), - "MODIFIED_DIR": relpath("modified_input"), - "ASSET_DIR": relpath("models"), # adjust if your folder name differs - # "ICON_DIR": relpath("intugle_assets"), - } - - for p in paths.values(): - p.mkdir(parents=True, exist_ok=True) - - # If intugle settings is available, set project base - try: - # 'settings' should be imported earlier from intugle.core.settings - settings.PROJECT_BASE = str(paths["ASSET_DIR"]) - except NameError: - # settings not imported/available; skip without failing the app - pass - - return paths - - -# ---- Call once near top of script ---- -dirs = init_io_dirs() -INPUT_DIR = dirs["INPUT_DIR"] -MODIFIED_DIR = dirs["MODIFIED_DIR"] -ASSET_DIR = dirs["ASSET_DIR"] -# ICON_DIR = dirs["ICON_DIR"] -# ----------------------- -# Session state initialization & lightweight router -# ----------------------- - -# Types (optional, for clarity in editors) -Route = Literal["home", "semantic"] - -DEFAULT_STATE = { - # Core app state - "uploader_key": 0, - "tables": {}, - "ingest_log": [], - "seen_files": {}, - "main_modify_select": "none", - "llm_choice": "openai", - "llm_config": {}, - - # Router + semantic flow - "route": "home", # type: Route - "semantic_table": None, - "data_profiling_glossary": False, - "semantic_model_done": False, - "creds_saved": False, - "links_list": None, - - # One-shot flags - "just_uploaded_rerun": False, -} - - -def init_session_state(defaults: dict) -> None: - """Idempotently seed st.session_state with defaults.""" - for k, v in defaults.items(): - st.session_state.setdefault(k, v) - - -# Initialize once at top of file -init_session_state(DEFAULT_STATE) - -# Clear one-shot flags if set -_ = st.session_state.pop("just_uploaded_rerun", False) - -# Convenience: whether to show the upload expander -upload_expander: bool = (st.session_state["uploader_key"] == 0) - -# --- init state (add once) --- - -# ---- sensible defaults so it doesn't crash on first run -# st.session_state.setdefault("seen_files", {}) -# st.session_state.setdefault("creds_saved", False) -# st.session_state.setdefault("route", "home") -# st.session_state.setdefault("data_profiling_glossary", False) -# st.session_state.setdefault("semantic_model_done", False) - -# # ----- Session init ----- -# if "creds_saved" not in st.session_state: -# st.session_state.creds_saved = False - -# # Provider choice: "openai", "azure-openai", "gemini" -# if "llm_choice" not in st.session_state: -# st.session_state.llm_choice = "openai" - -# # Store normalized config for the active provider -# if "llm_config" not in st.session_state: -# st.session_state.llm_config = {} - - -# # Initialize state -# # if "uploader_key" not in st.session_state: st.session_state["uploader_key"] = 0 -# if "tables" not in st.session_state: st.session_state["tables"] = {} -# if "ingest_log" not in st.session_state: st.session_state["ingest_log"] = [] -# if "seen_files" not in st.session_state: st.session_state["seen_files"] = {} -# if "main_modify_select" not in st.session_state: st.session_state["main_modify_select"] = 'none' - -# # --- Simple router state --- -# if "route" not in st.session_state: -# st.session_state["route"] = "home" # "home" | "semantic" -# if "semantic_table" not in st.session_state: -# st.session_state["semantic_table"] = None -# if "data_profiling_glossary" not in st.session_state: -# st.session_state["data_profiling_glossary"] = False -# if "semantic_model_done" not in st.session_state: -# st.session_state["semantic_model_done"] = False - - -# # ---- init once (top of file) ---- -# st.session_state.setdefault("uploader_key", 0) -# st.session_state.setdefault("just_uploaded_rerun", False) - -# # If we just forced a rerun, clear the flag (one-shot) -# if st.session_state.pop("just_uploaded_rerun", False): -# pass - - -# if st.session_state["uploader_key"] ==0: -# upload_expander = True -# else: -# upload_expander = False - - -# ----------------------- -# UI: Header -# ----------------------- - -# download image -# if upload_expander: -# # url = "https://commons.wikimedia.org/wiki/File:Intugle_icon.png" -# url = "https://commons.wikimedia.org/wiki/File:Intugle_main_logo.png" -# path = download_image(url, save_folder=ICON_DIR) -# print(f"Image downloaded to: {path}") - -# Layout: logo + title side by side -_, tcol1, tcol2 = st.columns([1, 2, 0.5]) - -with tcol1: - st.title(":violet[Intugle Semantic Modeler]") - -with tcol2: - # st.link_button("🌐 Website", "https://intugle.ai/",type="tertiary") - st.link_button("GitHub Repo", "https://intugle.github.io/data-tools/") - - -st.logo( - # 'https://commons.wikimedia.org/wiki/File:Intugle_icon.png', - "intugle_assets/Intugle_main_logo.png", - size='large' -) - - -# ========================= -# Status Bar (single row) -# ========================= - -# ---- your booleans -upload_done = bool(st.session_state["seen_files"]) -creds_done = bool(st.session_state["creds_saved"]) -freeze_done = st.session_state.get("route") == "semantic_all" -profiling_done = bool(st.session_state["data_profiling_glossary"]) -links_done = bool(st.session_state["semantic_model_done"]) - -# ---- stages in order -stages = [ - ("Upload Files", upload_done), - ("LLM Credentials", creds_done), - ("Rename Tables & Columns", freeze_done), - ("Profiling & Glossary", profiling_done), - ("Semantic Link Identification", links_done), -] - -# ---- render in sidebar -with st.sidebar: - st.markdown("## Progress") - - st.markdown(""" -
-""", unsafe_allow_html=True) - - html_parts = [] - for label, done in stages: - icon = "✔️" if done else "🟣" - state_cls = "done" if done else "todo" - html_parts.append( - f'
{icon}{label}
' - ) - - st.markdown("".join(html_parts) + "
", unsafe_allow_html=True) - st.divider() -# ------------------------------------------ -# st.title("Intugle Semantic Modeler") - - -if st.session_state["route"] == "home": - - st.subheader("How it works") - - dot = Digraph( - name="intugle_flow", - graph_attr={ - "rankdir": "LR", # left → right - "bgcolor": "white", - "fontsize": "10", - "pad": "0.2", - }, - node_attr={ - "shape": "rect", - "style": "rounded,filled", - "fontname": "Inter, Helvetica, Arial, sans-serif", - "color": "#8B5CF6", # Intugle purple border - "fillcolor": "#F3E8FF", # soft lilac fill - "fontsize": "11", - }, - edge_attr={ - "color": "#8B5CF6", - "fontname": "Inter, Helvetica, Arial, sans-serif", - "fontsize": "10", - }, - ) - - # Nodes - dot.node("U", "Upload CSV/Excel\n(one or many)") - dot.node("S", "Size ≤ 25 MB?", shape="diamond", fillcolor="#EDE9FE") - dot.node("P", "Profile columns\n• Uniqueness\n• Completeness\n• Data type") - dot.node("G", "Generate glossary\nfor all columns") - dot.node("M", "Build semantic model\n(Currently single link per table pair)") - dot.node("X", "Skip file\n(too large)", fillcolor="#FFE4E6", color="#EF4444") - - # Edges - dot.edge("U", "S") - dot.edge("S", "P", label="Yes") - dot.edge("S", "X", label="No") - dot.edge("P", "G") - dot.edge("G", "M") - - # Render in Streamlit - st.graphviz_chart(dot, width="stretch") - - st.subheader("1) Upload Files") - with st.expander("Upload Your Files here..", expanded=upload_expander): - - uploaded_files = st.file_uploader( - "Drop files here or browse", - type=[ext.lstrip(".") for ext in ACCEPTED_TYPES], - accept_multiple_files=True, - key=f"uploader_{st.session_state['uploader_key']}",) - - if uploaded_files: - st.caption(f"Received {len(uploaded_files)} file(s).") - - for upl in uploaded_files: - # Read bytes once so we can hash and parse from the same buffer - file_bytes = upl.getvalue() - size_mb = sizeof_mb(len(file_bytes)) - sha = hashlib.sha1(file_bytes).hexdigest() - - # Skip files already seen by content hash (prevents duplicates on reruns) - if sha in st.session_state["seen_files"]: - continue - - # Build a log row; we only append if we actually process/skip/error - log_row = { - "source_file": upl.name, - "size_mb": size_mb, - "status": "⏳ pending", - "table_name": None, - "rows": None, - "cols": None, - "details": "", - } - - # Enforce size - if size_mb > MAX_MB: - st.warning(f"Skipping **{upl.name}** because it exceeds {MAX_MB} MB.") - # log_row["status"] = "⛔ too large (>25MB)" - # log_row["details"] = "Skipped due to size limit." - # st.session_state["ingest_log"].append(log_row) - continue - - # Parse - try: - with st.spinner(f"Reading `{upl.name}`..."): - df, note = read_bytes_to_df(upl.name, file_bytes) - except Exception as e: - st.error(f"Failed to read **{upl.name}**: {e}") - # log_row["status"] = "❌ read error" - # log_row["details"] = str(e) - # st.session_state["ingest_log"].append(log_row) - continue - - # Standardize columns - with st.spinner(f"Standardizing columns for `{upl.name}`..."): - df = standardize_columns(df) - - # # Initial table name (no extra counters unless a different file collides) - input_path = INPUT_DIR / safe_filename(clean_table_name(upl.name), '.csv') - try: - if (not input_path.exists()) or (hashlib.sha1(input_path.read_bytes()).hexdigest() != sha): - input_path.write_bytes(file_bytes) - except Exception as e: - st.warning(f"Could not save original file for **{upl.name}**: {e}") - - # b) Standardized table -> modified_input/ as CSV (ALWAYS overwrite) - default_name = clean_table_name(upl.name) - unique_default = ensure_unique_name(default_name, st.session_state["tables"]) - - modified_path = MODIFIED_DIR / safe_filename(unique_default, ".csv") - try: - df.to_csv(modified_path, index=False) - # cleanup any legacy xlsx with the same stem - legacy_xlsx = (MODIFIED_DIR / safe_filename(unique_default, ".xlsx")) - if legacy_xlsx.exists(): - legacy_xlsx.unlink(missing_ok=True) - except Exception as e: - st.error(f"Failed to save standardized CSV to modified_input/: {e}") - continue - # ----------------------------------------------- - - # ----------------------------------------------- - # Metadata - rows_i, cols_i = int(df.shape[0]), int(df.shape[1]) - meta = { - "source_file": upl.name, - "size_mb": size_mb, - "note": note, - "rows": rows_i, - "cols": cols_i, - "timestamp": time.time(), - "modified_path": str(modified_path), # <— new - } - - # Save table and fingerprint - with st.spinner(f"Saving `{unique_default}`..."): - add_table_to_state(unique_default, df, meta) - - st.session_state["seen_files"][sha] = { - "table_name": unique_default, - "source_file": upl.name, - "size_mb": size_mb, - } - - # Log success - log_row.update( - { - "status": "✅ loaded", - "table_name": unique_default, - "rows": rows_i, - "cols": cols_i, - "details": note or "", - } - ) - st.session_state["ingest_log"].append(log_row) - - st.toast("Processing completed for all eligible files.") - # Clear the uploader selection and force exactly one rerun - st.session_state["uploader_key"] += 1 # resets file_uploader - st.session_state["just_uploaded_rerun"] = True # one-shot marker - st.rerun() - - # ----------------------- - # Tables overview (read-only summary) - # ----------------------- - # Normalize ingest_log names from seen_files (ensures latest rename is shown) - for d, info in st.session_state.get("seen_files", {}).items(): - latest_name = info.get("table_name") - for row in st.session_state.get("ingest_log", []): - if row.get("source_file") == info.get("source_file") and row.get("status", "").startswith("✅"): - row["table_name"] = latest_name - - if st.session_state.get("tables") or st.session_state.get("ingest_log"): - with st.expander("1) Review Tables", expanded=False): - - # Summary now shows status/details only; no inline actions - if st.session_state.get("ingest_log"): - summary_df = pd.DataFrame(st.session_state["ingest_log"]).copy() - cols_order = ["table_name", "source_file", "size_mb", "rows", "cols", "status", "details"] - summary_df = summary_df[[c for c in cols_order if c in summary_df.columns]] - else: - # Fallback if ingest_log not present - rows = [] - for tname, payload in st.session_state["tables"].items(): - m = payload["meta"] - rows.append({ - "table_name": tname, - "source_file": m["source_file"], - "size_mb": m["size_mb"], - "rows": m["rows"], - "cols": m["cols"], - "status": "✅ loaded", - "details": m.get("note", ""), - }) - summary_df = pd.DataFrame(rows).sort_values("table_name") - - st.dataframe(summary_df, width="stretch", hide_index=True) - else: - # st.info("No tables loaded yet. Upload some CSV/Excel files above to get started!") - - st.toast(""" - **Welcome to Intugle!** - We’re thrilled to have you here. - """, icon='🕸️') - - # ----------------------- - # Modify table & columns (MAIN PAGE, under the review table) - # ----------------------- - - # --- Always-visible table picker + synced final name --- - - if not st.session_state.get("tables"): - # st.info("Load tables to modify them here.") - time.sleep(1) - st.toast("Let us Start by uploading files and submit your LLM credentials") - - else: - st.divider() - st.subheader("2) Modify table & columns") - # Ensure state keys exist - st.session_state.setdefault("final_name_user_edited", False) - - # left, right = st.columns([0.5, 0.5]) - - # Build options; always render the selectbox - has_tables = bool(st.session_state.get("tables")) - tnames = sorted(st.session_state["tables"].keys()) if has_tables else [] - options = tnames if has_tables else ["— no tables loaded —"] - col1, col2, col3, col4 = st.columns([2, 0.5, 2, 2]) - with col1: - sel = st.selectbox( - "Select table to modify", - options=options, - index=0, - # key="main_modify_select", - on_change=_on_select_change, - disabled=not has_tables, - help="Choose a loaded table. This is always shown; it will be disabled until you upload data." - ) - out_fmt = "CSV" - # Or, if you want to show a control but keep a default, uncomment: - # out_fmt = st.radio("Save format", ["CSV", "Excel"], horizontal=True, key="main_modify_fmt") - - # Guard when there are no tables yet - if not has_tables: - st.info("Load tables to modify them here.") - st.stop() - - # Use the selected table - payload = st.session_state["tables"][sel] - df_current = payload["df"] - orig_cols = list(df_current.columns) - with col2: - # add a right emoji arrow - # st.markdown("
➡️
", unsafe_allow_html=True) - st.image( - "https://upload.wikimedia.org/wikipedia/commons/c/ca/Eo_circle_purple_arrow-right.svg", - width=80, # Manually Adjust the width of the image as per requirement - ) - - with col3: - # Final name input that mirrors selection until the user edits it - # st.markdown("**Rename table (optional)**") - final_name_raw = st.text_input( - "Rename table (optional)", - # key="main_modify_final_name", - # value=st.session_state.get("main_modify_final_name", sel) or sel, - value=sel, - on_change=_on_name_edit, - help="Defaults to the selected table name. Will be normalized to lowercase with underscores." - ) - - # Current paths - current_name = sel - current_path = _csv_path_for(current_name, MODIFIED_DIR) - - # ------------------------------- - # ACTIONS - # ------------------------------- - __, _, act_col1, ___, = st.columns([2, 0.5, 2, 2]) - - # 1) RENAME TABLE (and CSV file) — no column changes here - with act_col1: - if st.button("Rename table", type="primary", width="content", key="btn_rename_table"): - new_name = _normalized_table_name(final_name_raw) - if not new_name: - st.error("Final table name cannot be empty after normalization.") - st.stop() - - if new_name == current_name: - st.toast("Table name unchanged.") - else: - new_path = _csv_path_for(new_name, MODIFIED_DIR) - - # Move/rename the CSV on disk (if it exists). If it doesn't, create from in-memory df. - try: - if current_path.exists(): - # Avoid cross-device rename oddities - current_path.replace(new_path) - else: - # If the file isn't there yet, write the current DataFrame - st.session_state["tables"][current_name]["df"].to_csv(new_path, index=False) - except Exception as e: - st.error(f"Failed to rename CSV file: {e}") - st.stop() - - # Cleanup any legacy .xlsx with the old name - legacy_xlsx = MODIFIED_DIR / safe_filename(current_name, ".xlsx") - legacy_xlsx.unlink(missing_ok=True) - - # Update in-memory state & logs - rename_table_in_state(current_name, new_name) - st.session_state["tables"][new_name]["meta"]["modified_path"] = str(new_path) - for row in st.session_state.get("ingest_log", []): - if row.get("table_name") == current_name: - row["table_name"] = new_name - for _, info in st.session_state.get("seen_files", {}).items(): - if info.get("table_name") == current_name: - info["table_name"] = new_name - - # Sync the selectbox and input defaults - st.session_state["main_modify_select"] = new_name - st.session_state["main_modify_final_name"] = new_name - st.session_state["final_name_user_edited"] = False - - st.success(f"Renamed **{current_name}** → **{new_name}** and updated CSV file.") - st.rerun() - - # Column editor (unchanged) - st.markdown("**Edit or ignore columns**") - editor_src = { - "original_column_name": orig_cols, - "new_column_name": orig_cols.copy(), # default to current names - "ignore_column": [False] * len(orig_cols), - } - edit_df = st.data_editor( - pd.DataFrame(editor_src), - hide_index=True, - width="stretch", - column_config={ - "original_column_name": st.column_config.TextColumn("Original_Column_Name", disabled=True), - "new_column_name": st.column_config.TextColumn("New_Column_Name (Editable)", help="Enter desired column names"), - "ignore_column": st.column_config.CheckboxColumn("Select_column_to_Ignore", help="Drop this column for sematic model"), - }, - key="main_modify_editor", - ) - # ---------- Editor UI already created above ---------- - # sel, df_current, orig_cols, final_name_raw, edit_df exist - - # 2) FREEZE COLUMN NAMES — apply edits/ignores & save back to SAME file name (CSV overwrite) - # with act_col2: - if st.button("Freeze column names", type="primary", width="content", key="btn_freeze_cols"): - # Pull edits - new_names_input = list(edit_df["new_column_name"]) - ignored_flags = list(edit_df["ignore_column"]) - - # Validate proposed names - errs = validate_column_names(orig_cols, new_names_input, ignored_flags) - if errs: - for e in errs: - st.error(e) - st.stop() - - # Build modified DataFrame - keep_idx = [i for i, ig in enumerate(ignored_flags) if not ig] - kept_cols = [orig_cols[i] for i in keep_idx] - new_names = [normalize_col_name(new_names_input[i]) for i in keep_idx] - - df_mod = df_current.loc[:, kept_cols].copy() - df_mod.columns = new_names - - # Determine the *current* table name (which might have been renamed already) - # live_name = st.session_state["main_modify_select"] - live_name = final_name_raw - st.info(live_name) - live_path = _csv_path_for(live_name, MODIFIED_DIR) - - # Save back to the SAME file (overwrite) - try: - df_mod.to_csv(live_path, index=False) - except Exception as e: - st.error(f"Failed to write CSV: {e}") - st.stop() - time.sleep(3) - # Update in-memory table payload - st.session_state["tables"][live_name]["df"] = df_mod - st.session_state["tables"][live_name]["meta"]["rows"] = int(df_mod.shape[0]) - st.session_state["tables"][live_name]["meta"]["cols"] = int(df_mod.shape[1]) - st.session_state["tables"][live_name]["meta"]["modified_path"] = str(live_path) - - st.success(f"Column names frozen and saved to `{live_path.name}`.") - st.rerun() - - -# ----------------------- -# Sidebar -# ----------------------- -with st.sidebar: - st.header("🔑 LLM Settings") - - # If already saved this session: show summary + "Change settings" - if st.session_state.creds_saved: - # st.write(f"**LLM Service:** `{provider_display_string()}`") - - choice = st.session_state.llm_choice - cfg = st.session_state.llm_config - - if choice == "openai": - # st.write(f"**OpenAI Model Loaded** `{cfg.get('model','')}`") - # st.write(f"**Model:** `{cfg.get('model','')}`") - # st.write(f"**OPENAI_API_KEY:** {mask(get_secret('OPENAI_API_KEY'))}") - # pip install -U langchain-openai - - llm_temp = ChatOpenAI( - model=os.environ["LLM_PROVIDER"], # pick your OpenAI model - temperature=0, - # api_key="...", # or set env var: OPENAI_API_KEY=... - # base_url="...", # only if you're using a compatible proxy - ) - - messages = [ - ("system", "You are a helpful assistant that translates French to English. Translate the user sentence."), - ("human", "Le point de terminaison OpenAI a été chargé avec succès."), - ] - - ai_msg = llm_temp.invoke(messages) - st.success(ai_msg.content) - - # st.success(f"**OpenAI API Key load successful:**") - - elif choice == "azure-openai": - # Import Azure OpenAI - llm_temp = AzureChatOpenAI( - model=os.environ["LLM_PROVIDER"], # your deployed model name or alias - # azure_endpoint="https://.openai.azure.com/", - # api_key="...", - # api_version="2024-xx-xx" - ) - messages = [ - ( - "system", - "You are a helpful assistant that translates French to English. Translate the user sentence.", - ), - ("human", "Le point de terminaison Azure OpenAI a été chargé avec succès."), - ] - ai_msg = llm_temp.invoke(messages) - ai_msg = ai_msg.content - # st.success(f"**Azure OpenAI Endpoint load successful:**") - st.success(ai_msg) - # st.write(f'LLM_PROVIDER: {os.environ["LLM_PROVIDER"]}') - # st.write(f'AZURE_OPENAI_API_KEY: {os.environ["AZURE_OPENAI_API_KEY"]}') - # st.write(f'OPENAI_API_VERSION: {os.environ["OPENAI_API_VERSION"]}') - # st.write(f'AZURE_OPENAI_ENDPOINT: {os.environ["AZURE_OPENAI_ENDPOINT"]}') - - elif choice == "gemini": - # st.write(f"**Model:** `{cfg.get('model','')}`") - # Common env names for Gemini: GEMINI_API_KEY or GOOGLE_API_KEY - # st.write(f"**GEMINI/GOOGLE_API_KEY:** {mask(get_secret('GEMINI_API_KEY') or get_secret('GOOGLE_API_KEY'))}") - - llm_temp = ChatGoogleGenerativeAI( - model=os.environ["LLM_PROVIDER"], - temperature=0, - # google_api_key="...", # or set env var: GOOGLE_API_KEY=... - ) - - messages = [ - ("system", "You are a helpful assistant that translates French to English. Translate the user sentence."), - ("human", "Le point de terminaison Google Gemini a été chargé avec succès."), - ] - - ai_msg = llm_temp.invoke(messages) - ai_msg = ai_msg.content - st.success(ai_msg) - # st.success(f"**Gemini API Key load successful:**") - # if st.button("Change settings"): - # st.session_state.creds_saved = False - # st.stop() - - # Otherwise: render the provider picker + provider-specific fields - with st.expander("LLM Settings", expanded=not st.session_state.creds_saved): - provider = st.selectbox( - "Choose LLM provider", - options=["openai", "azure-openai", "google_genai"], - index=["openai", "azure-openai", "google_genai"].index(st.session_state.llm_choice), - help="Pick your LLM backend." - ) - st.session_state.llm_choice = provider - - # --------- OPENAI --------- - if provider == "openai": - # Pre-fill from env/secrets if present - pre_key = get_secret("OPENAI_API_KEY", "") - model = st.selectbox( - "Model", - # ["openai:gpt-3.5-turbo"], - ["gpt-4o-mini", "gpt-4o", "gpt-3.5-turbo"], - index=0 - ) - api_key = st.text_input( - "OpenAI API key", - type="password", - value=pre_key, - placeholder="sk-********************************" - ) - - if st.button("Save provider settings", type="primary", width="stretch"): - if not api_key or len(api_key) < 20: - st.error("Please enter a valid OpenAI API key.") - else: - st.session_state.llm_config = { - "model": model, - } - # Normalize env vars (and a generic LLM_PROVIDER string if you use it elsewhere) - # save_env({ - # "OPENAI_API_KEY": api_key.strip(), - # "LLM_PROVIDER": f"openai:{model}", - # }) - settings.LLM_PROVIDER = f"openai:{model}" - os.environ["OPENAI_API_KEY"] = api_key.strip() - os.environ["LLM_PROVIDER"] = model - st.session_state.creds_saved = True - st.success("OpenAI settings saved.") - st.rerun() - - # --------- AZURE OPENAI --------- - elif provider == "azure-openai": - pre_key = get_secret("AZURE_OPENAI_API_KEY", "") - pre_endpoint = get_secret("AZURE_OPENAI_ENDPOINT", "https://YOUR-RESOURCE-NAME.openai.azure.com/") - pre_llm_provider = get_secret("LLM_PROVIDER", "") - pre_version = get_secret("OPENAI_API_VERSION", "2024-06-01") - - endpoint = st.text_input("Azure OpenAI endpoint", value=pre_endpoint, placeholder="https://.openai.azure.com/") - llm_provider = st.text_input("Provider name", value=pre_llm_provider, placeholder="azure_openai:gpt-4o") - api_version = st.text_input("API version", value=pre_version, placeholder="2024-06-01") - api_key = st.text_input("Azure OpenAI API key", type="password", value=pre_key, placeholder="****************") - - if st.button("Save provider settings", type="primary", width="stretch"): - errs = [] - if not endpoint.startswith("http"): - errs.append("Endpoint must start with http/https.") - if not llm_provider: - errs.append("LLM provider name is required.") - if not api_version: - errs.append("API version is required.") - if not api_key or len(api_key) < 20: - errs.append("Enter a valid Azure OpenAI API key.") - if errs: - for e in errs: st.error(e) - else: - st.session_state.llm_config = { - "endpoint": endpoint.strip(), - "llm_provider": llm_provider.strip(), - "api_version": api_version.strip(), - } - # save_env({ - # "AZURE_OPENAI_ENDPOINT": endpoint.strip(), - # "LLM_PROVIDER": llm_provider.strip(), - # "OPENAI_API_VERSION": api_version.strip(), - # "AZURE_OPENAI_API_KEY": api_key.strip(), - # # "LLM_PROVIDER": f"azure-openai:{deployment.strip()}@{api_version.strip()}", - # }) - os.environ["LLM_PROVIDER"] = str(llm_provider.strip()) - settings.LLM_PROVIDER = str(llm_provider.strip()) - os.environ["AZURE_OPENAI_API_KEY"] = str(api_key.strip()) - os.environ["OPENAI_API_VERSION"] = str(api_version.strip()) - os.environ["AZURE_OPENAI_ENDPOINT"] = str(endpoint.strip()) - - st.session_state.creds_saved = True - st.success("Azure OpenAI settings saved.") - st.rerun() - - # --------- GEMINI --------- - elif provider == "google_genai": - # Gemini keys are commonly under GEMINI_API_KEY or GOOGLE_API_KEY - pre_key = get_secret("GOOGLE_API_KEY") or "" - model = st.selectbox( - "Model", - ["gemini-2.5-pro", "gemini-2.5-flash", "gemini-2.5-flash-lite"], - index=0 - ) - api_key = st.text_input("Gemini API key", type="password", value=pre_key, placeholder="********************************") - - if st.button("Save provider settings", type="primary", width="stretch"): - if not api_key or len(api_key) < 20: - st.error("Please enter a valid Gemini API key.") - else: - st.session_state.llm_config = {"model": model} - # Set both common env var names so whichever client you use can find it - # save_env({ - # "GEMINI_API_KEY": api_key.strip(), - # "GOOGLE_API_KEY": api_key.strip(), - # "LLM_PROVIDER": f"gemini:{model}", - # }) - os.environ["GOOGLE_API_KEY"] = api_key.strip() - os.environ["LLM_PROVIDER"] = model - settings.LLM_PROVIDER = f"google_genai:{model}" - st.session_state.creds_saved = True - st.success("Gemini settings saved.") - st.rerun() - -with st.sidebar: - - # Settings & info - # with st.expander("Reset Settings",expanded=False): - # st.write(f"Max file size per file: **{MAX_MB} MB**") - # st.write(f"Accepted types: {', '.join(ACCEPTED_TYPES)}") - - if st.button("Reset App", type="primary", width="content"): - # st.session_state["tables"] = {} - # st.session_state["ingest_log"] = [] # optional: reset the log - # st.session_state["seen_files"] = {} # important: allow re-uploading same files - # st.session_state["uploader_key"] += 1 # <-- resets the uploader selection - clear_cache() - try: - shutil.rmtree(INPUT_DIR) - shutil.rmtree(ASSET_DIR) - shutil.rmtree(MODIFIED_DIR) - - print('input & modified_input folders found & deleted. You are good to go') - except: - print('No folders found. You are good to go') - - st.success("Cleared all loaded tables and upload selection.") - - st.rerun() - - -# ----------------------- ----------------------- ----------------------- ----------------------- ----------------------- -# 5) Create a Semantic Model (uses all files in modified_input/) -# ----------------------- ----------------------- ----------------------- ----------------------- ----------------------- -# ====================== - -# --- tiny helpers (define once) --- - -def _list_modified_files(): - return sorted(list(MODIFIED_DIR.glob("*.csv")) + list(MODIFIED_DIR.glob("*.xlsx"))) - - -def _source_type_from_suffix(p: Path) -> str: - return "csv" if p.suffix.lower() == ".csv" else "excel" - - -# Find modified files -modified_csv = list(MODIFIED_DIR.glob("*.csv")) -modified_xlsx = list(MODIFIED_DIR.glob("*.xlsx")) -modified_files = modified_csv + modified_xlsx -n_files = len(modified_files) - -# LLM readiness -ok, err = llm_ready_check() -if not ok: - st.error(err) - st.stop() - - -if n_files == 0: - st.info("No tables found. Upload & Freeze at least one table to continue.") -elif st.session_state.get("route") != "semantic_all": - st.caption(f"Found **{n_files}** file(s)") - # Brief preview of filenames (first 5) - show = [p.name for p in modified_files[:5]] - # st.write(", ".join(show) + (" ..." if n_files > 5 else "")) - - # Full-width action button - st.divider() - if st.button("🧠 Create Semantic Model from Modified Data", type="primary", width="stretch", key="btn_semantic_all"): - go_semantic_all() - - -# ========================= -# Semantic page content -# ========================= -if st.session_state.get("route") == "semantic_all": - # if st.sidebar.button("⬅️ Back to Data", key="btn_semantic_back"): - # st.session_state["route"] = "home" - # st.rerun() - ############################################################################## - with st.sidebar: - # Make the sidebar a full-height flex column - st.markdown(""" - - """, unsafe_allow_html=True) - - # --- your existing sidebar content goes here --- - st.markdown("# ") - # ... (status pills, etc.) - - # This flexible spacer expands and pushes what follows to the bottom - st.markdown('
', unsafe_allow_html=True) - - st.divider() - if st.button("⬅️ Back to Data", key="btn_semantic_back", width="stretch"): - st.session_state["route"] = "home" - st.rerun() - ############################################################################## - st.header("3) 🧠 Build Semantic Model") - # 2) Define domain BEFORE using it - st.markdown("#### Domain") - dcol1, dcol2 = st.columns([1, 1]) - with dcol1: - # show whatever is currently saved as the default in the box - domain = st.text_input( - "Enter a domain (e.g., Manufacturing)", - value=st.session_state.get("semantic_domain", "NA"), - key="semantic_domain_input", - ) - - # with dcol2: - if st.button("Set Domain", type="primary", width="content"): - # Only commit when the button is clicked - committed = (st.session_state.get("semantic_domain_input") or "").strip() or "NA" - st.session_state["semantic_domain"] = committed - st.toast(f"Domain set to: **{committed}**") - - # Require domain - - if not domain.strip() or domain.strip().upper() == "NA" or domain.strip() == "" or not bool(re.search(r"[A-Za-z0-9]", domain.strip())): - st.warning("Please enter a valid domain before building the semantic model.") - st.stop() - # 1) Define files BEFORE using them - files = _list_modified_files() - if not files: - st.warning("No files found in modified_input/. Go back and save at least one modified table.") - if st.button("⬅️ Back"): - st.session_state["route"] = "home" - st.rerun() - st.stop() - - st.caption(f"Semantic Model will be built for **{len(files)}** table(s).") - - gdf_iter_ph = st.empty() - # optional: keep a cumulative copy if you still want it elsewhere - st.session_state.setdefault("semantic_gdf_all", None) - - total = len(files) - my_bar = st.progress(0, text="Preparing data for semantic model creation. Please wait.") - - for idx, p in enumerate(files, start=1): # start=2 because first file already processed - if idx == 1: - # Initialize with the first file - first = files[0] - data_sources = {first.stem: {"path": str(first), "type": _source_type_from_suffix(first)}} - - with st.spinner(f"Initializing Semantic Model with `{first.name}` …"): - sm = SemanticModel(data_input=data_sources, domain=domain.strip()) - st.session_state["sm"] = sm - - ds_name = p.stem - - with st.spinner(f"Adding dataset `{ds_name}` …"): - new_ds = DataSet({"path": str(p), "type": _source_type_from_suffix(p)}, name=ds_name) - sm.datasets[ds_name] = new_ds - - with st.spinner(f"Profiling `{ds_name}` ({idx}/{total})…"): - sm.profile() - with st.spinner(f"Generating glossary for `{ds_name}` …"): - sm.generate_glossary() - - # with st.status(f"Preparing data to build semantic model `{ds_name}` …", expanded=True) as status: - # new_ds = DataSet({"path": str(p), "type": _source_type_from_suffix(p)}, name=ds_name) - # sm.datasets[ds_name] = new_ds - # st.write(f"Profiling `{ds_name}` ({idx}/{total}) …") - # sm.profile() - # st.write(f"Generating glossary for `{ds_name}` …") - # sm.generate_glossary() - percent_complete = (idx / total) - my_bar.progress(percent_complete, text=f"Preparing data for semantic model creation {round(percent_complete * 100)}%. Please wait.") - - try: - profiling_df = sm.profiling_df.copy() - glossary_df = sm.glossary_df.copy() - - gdf_this = pd.merge( - glossary_df, - profiling_df, - on=["table_name", "column_name"], - how="left", - suffixes=("_gloss", "_prof"), - ) - - # ---- FLUSH previous view and show ONLY this iteration ---- - gdf_iter_ph.empty() - with gdf_iter_ph.container(): - st.toast(f"##### ✅ Glossary for `{ds_name}` ({idx}/{total})") - # add title to the table - st.markdown(f"### Glossary & Profiling details for `{idx}` tables") - st.dataframe(gdf_this[['table_name', 'column_name', 'datatype_l1', 'datatype_l2', 'count', 'null_count', 'distinct_count', - 'uniqueness', 'completeness', 'business_glossary', 'business_tags', 'sample_data']], - hide_index=True, width="stretch") - st.session_state["sm"] = sm - - except Exception as e: - st.warning(f"Could not render glossary view for `{ds_name}`: {e}") - - st.success("Semantic model profiling & glossary generation completed for all tables.") - if not st.session_state["data_profiling_glossary"]: - st.session_state["data_profiling_glossary"] = True - st.rerun() - - # Link prediction and back button (unchanged) - st.divider() - st.subheader("4) Link Prediction") - - if "sm" not in st.session_state: - st.info("Build the semantic model first to enable link prediction.") - else: - sm = st.session_state["sm"] - if st.button("🔗 Run Link Prediction", type="primary", key="btn_link_pred", width="stretch"): - with st.spinner("Predicting links …"): - sm.predict_links() - predictor_instance = sm.link_predictor - st.session_state["links_list"] = predictor_instance.links # <-- SAVE to state - st.session_state["semantic_model_done"] = True # <-- FLAG done - st.success("Link prediction completed.") - - # --- render results if we have them --- - if st.session_state.get("semantic_model_done") and st.session_state.get("links_list"): - try: - show_links_graph_and_table(st.session_state["links_list"]) # <-- use saved list - except Exception as e: - st.warning(f"Could not render link prediction results: {e}") - # if st.button("🔗 Run Link Prediction", type="primary", width="stretch", key="btn_link_pred"): - # with st.spinner("Predicting links …"): - # sm.predict_links() - # # After running the pipeline... - # predictor_instance = sm.link_predictor - - # # Now you can use all the methods of the LinkPredictor - # links_list = predictor_instance.links - # st.success("Link prediction completed.") - - # try: - # # ---------- usage ---------- - # show_links_graph_and_table(links_list) # <-- pass your list[PredictedLink] here - # except Exception as e: - # st.warning(f"Could not render link prediction results: {e}") - - # if st.session_state["semantic_model_done"] == False: - # st.session_state["semantic_model_done"] = True - # st.rerun() - - with st.sidebar: - zip_bytes, file_count = build_yaml_zip(ASSET_DIR) - if file_count: - st.download_button( - label=f"Download {file_count} YAML file{'s' if file_count != 1 else ''} (ZIP)", - data=zip_bytes, - file_name=f"yaml_assets_{time.strftime('%Y%m%d_%H%M%S')}.zip", - mime="application/zip", - width="stretch", - ) - else: - st.info(f"No .yml or .yaml files found in '{ASSET_DIR}/'.") - - diff --git a/tests/adapters/test_databricks_adapter.py b/tests/adapters/test_databricks_adapter.py index 9588d96..c5c05c9 100644 --- a/tests/adapters/test_databricks_adapter.py +++ b/tests/adapters/test_databricks_adapter.py @@ -224,27 +224,6 @@ def test_rejects_non_databricks_configs(self): for invalid in invalid_configs: assert can_handle_databricks(invalid) is False - def test_type_field_not_strictly_enforced(self): - """Test that Pydantic validates structure but doesn't enforce type value. - - Note: DatabricksConfig validates that 'identifier' field exists, - but doesn't strictly validate the 'type' field value. - This means a config with wrong type but valid structure passes validation. - - This could be a potential bug - consider adding field validation. - """ - # This passes validation because it has the required 'identifier' field - config_with_wrong_type = {"identifier": "table", "type": "snowflake"} - - # Pydantic validates successfully (structure is correct) - result = DatabricksAdapter.check_data(config_with_wrong_type) - - # But the type field is NOT corrected - it keeps the wrong value! - # This is a potential bug in the validation logic - assert result.type == "snowflake" # Wrong type passes through! - assert result.identifier == "table" - - # ============================================================================ # Error Handling Tests # ============================================================================ diff --git a/tests/adapters/test_snowflake_adapter.py b/tests/adapters/test_snowflake_adapter.py index da032e8..fa84018 100644 --- a/tests/adapters/test_snowflake_adapter.py +++ b/tests/adapters/test_snowflake_adapter.py @@ -211,25 +211,6 @@ def test_rejects_non_snowflake_configs(self): for invalid in invalid_configs: assert can_handle_snowflake(invalid) is False - def test_type_field_not_strictly_enforced(self): - """Test that Pydantic validates structure but doesn't enforce type value. - - Note: SnowflakeConfig validates that 'identifier' field exists, - but doesn't strictly validate the 'type' field value. - This means a config with wrong type but valid structure passes validation. - - This could be a potential bug - consider adding field validation. - """ - # This passes validation because it has the required 'identifier' field - config_with_wrong_type = {"identifier": "table", "type": "databricks"} - - # Pydantic validates successfully (structure is correct) - result = SnowflakeAdapter.check_data(config_with_wrong_type) - - # But the type field is NOT corrected - it keeps the wrong value! - # This is a potential bug in the validation logic - assert result.type == "databricks" # Wrong type passes through! - assert result.identifier == "table" # ============================================================================ diff --git a/uv.lock b/uv.lock index 5af6dea..2823b65 100644 --- a/uv.lock +++ b/uv.lock @@ -128,6 +128,22 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/fb/76/641ae371508676492379f16e2fa48f4e2c11741bd63c48be4b12a6b09cba/aiosignal-1.4.0-py3-none-any.whl", hash = "sha256:053243f8b92b990551949e63930a839ff0cf0b0ebbe0597b0f3fb19e1a0fe82e", size = 7490, upload-time = "2025-07-03T22:54:42.156Z" }, ] +[[package]] +name = "altair" +version = "5.5.0" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "jinja2" }, + { name = "jsonschema" }, + { name = "narwhals" }, + { name = "packaging" }, + { name = "typing-extensions", marker = "python_full_version < '3.14'" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/16/b1/f2969c7bdb8ad8bbdda031687defdce2c19afba2aa2c8e1d2a17f78376d8/altair-5.5.0.tar.gz", hash = "sha256:d960ebe6178c56de3855a68c47b516be38640b73fb3b5111c2a9ca90546dd73d", size = 705305, upload-time = "2024-11-23T23:39:58.542Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/aa/f3/0b6ced594e51cc95d8c1fc1640d3623770d01e4969d29c0bd09945fafefa/altair-5.5.0-py3-none-any.whl", hash = "sha256:91a310b926508d560fe0148d02a194f38b824122641ef528113d029fcd129f8c", size = 731200, upload-time = "2024-11-23T23:39:56.4Z" }, +] + [[package]] name = "annotated-types" version = "0.7.0" @@ -277,6 +293,15 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/b9/fa/123043af240e49752f1c4bd24da5053b6bd00cad78c2be53c0d1e8b975bc/backports.tarfile-1.2.0-py3-none-any.whl", hash = "sha256:77e284d754527b01fb1e6fa8a1afe577858ebe4e9dad8919e34c862cb399bc34", size = 30181, upload-time = "2024-05-28T17:01:53.112Z" }, ] +[[package]] +name = "blinker" +version = "1.9.0" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/21/28/9b3f50ce0e048515135495f198351908d99540d69bfdc8c1d15b73dc55ce/blinker-1.9.0.tar.gz", hash = "sha256:b4ce2265a7abece45e7cc896e98dbebe6cead56bcf805a3d23136d145f5445bf", size = 22460, upload-time = "2024-11-08T17:25:47.436Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/10/cb/f2ad4230dc2eb1a74edf38f1a38b9b52277f75bef262d8908e60d957e13c/blinker-1.9.0-py3-none-any.whl", hash = "sha256:ba0efaa9080b619ff2f3459d1d500c57bddea4a6b424b60a91141db6fd2f08bc", size = 8458, upload-time = "2024-11-08T17:25:46.184Z" }, +] + [[package]] name = "boto3" version = "1.40.43" @@ -1329,6 +1354,30 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/ee/45/b82e3c16be2182bff01179db177fe144d58b5dc787a7d4492c6ed8b9317f/frozenlist-1.7.0-py3-none-any.whl", hash = "sha256:9a5af342e34f7e97caf8c995864c7a396418ae2859cc6fdf1b1073020d516a7e", size = 13106, upload-time = "2025-06-09T23:02:34.204Z" }, ] +[[package]] +name = "gitdb" +version = "4.0.12" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "smmap" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/72/94/63b0fc47eb32792c7ba1fe1b694daec9a63620db1e313033d18140c2320a/gitdb-4.0.12.tar.gz", hash = "sha256:5ef71f855d191a3326fcfbc0d5da835f26b13fbcba60c32c21091c349ffdb571", size = 394684, upload-time = "2025-01-02T07:20:46.413Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/a0/61/5c78b91c3143ed5c14207f463aecfc8f9dbb5092fb2869baf37c273b2705/gitdb-4.0.12-py3-none-any.whl", hash = "sha256:67073e15955400952c6565cc3e707c554a4eea2e428946f7a4c162fab9bd9bcf", size = 62794, upload-time = "2025-01-02T07:20:43.624Z" }, +] + +[[package]] +name = "gitpython" +version = "3.1.45" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "gitdb" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/9a/c8/dd58967d119baab745caec2f9d853297cec1989ec1d63f677d3880632b88/gitpython-3.1.45.tar.gz", hash = "sha256:85b0ee964ceddf211c41b9f27a49086010a190fd8132a24e21f362a4b36a791c", size = 215076, upload-time = "2025-07-24T03:45:54.871Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/01/61/d4b89fec821f72385526e1b9d9a3a0385dda4a72b206d28049e2c7cd39b8/gitpython-3.1.45-py3-none-any.whl", hash = "sha256:8908cb2e02fb3b93b7eb0f2827125cb699869470432cc885f019b8fd0fccff77", size = 208168, upload-time = "2025-07-24T03:45:52.517Z" }, +] + [[package]] name = "google-ai-generativelanguage" version = "0.7.0" @@ -1392,6 +1441,15 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/86/f1/62a193f0227cf15a920390abe675f386dec35f7ae3ffe6da582d3ade42c7/googleapis_common_protos-1.70.0-py3-none-any.whl", hash = "sha256:b8bfcca8c25a2bb253e0e0b0adaf8c00773e5e6af6fd92397576680b807e0fd8", size = 294530, upload-time = "2025-04-14T10:17:01.271Z" }, ] +[[package]] +name = "graphviz" +version = "0.21" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/f8/b3/3ac91e9be6b761a4b30d66ff165e54439dcd48b83f4e20d644867215f6ca/graphviz-0.21.tar.gz", hash = "sha256:20743e7183be82aaaa8ad6c93f8893c923bd6658a04c32ee115edb3c8a835f78", size = 200434, upload-time = "2025-06-15T09:35:05.824Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/91/4c/e0ce1ef95d4000ebc1c11801f9b944fa5910ecc15b5e351865763d8657f8/graphviz-0.21-py3-none-any.whl", hash = "sha256:54f33de9f4f911d7e84e4191749cac8cc5653f815b06738c54db9a15ab8b1e42", size = 47300, upload-time = "2025-06-15T09:35:04.433Z" }, +] + [[package]] name = "greenlet" version = "3.2.4" @@ -1690,7 +1748,7 @@ wheels = [ [[package]] name = "intugle" -version = "1.0.11" +version = "1.0.12" source = { editable = "." } dependencies = [ { name = "aiofiles" }, @@ -1738,6 +1796,14 @@ snowflake = [ { name = "snowflake-snowpark-python", extra = ["pandas"] }, { name = "sqlglot" }, ] +streamlit = [ + { name = "graphviz" }, + { name = "plotly" }, + { name = "pyngrok" }, + { name = "python-dotenv" }, + { name = "streamlit" }, + { name = "xlsxwriter" }, +] [package.dev-dependencies] dev = [ @@ -1776,6 +1842,7 @@ requires-dist = [ { name = "databricks-sql-connector", marker = "extra == 'databricks'", specifier = ">=4.1.3" }, { name = "duckdb", specifier = ">=1.3.2" }, { name = "fastapi", extras = ["standard"], specifier = ">=0.116.1" }, + { name = "graphviz", marker = "extra == 'streamlit'" }, { name = "langchain", extras = ["anthropic", "google-genai", "openai"], specifier = ">=0.3.27,<1.0.0" }, { name = "langchain-community", specifier = ">=0.3.21,<=0.3.30" }, { name = "langchain-openai", specifier = ">=0.3.28,<=0.3.33" }, @@ -1786,12 +1853,15 @@ requires-dist = [ { name = "nltk", specifier = ">=3.9.1" }, { name = "numpy", specifier = "<=2.3.0" }, { name = "pandas", specifier = ">=2.2.2" }, + { name = "plotly", marker = "extra == 'streamlit'" }, { name = "pyaml", specifier = ">=25.7.0" }, { name = "pydantic", specifier = ">=2.11.7" }, { name = "pydantic-settings", specifier = ">=2.10.1" }, { name = "pyfunctional", specifier = ">=1.5.0" }, + { name = "pyngrok", marker = "extra == 'streamlit'", specifier = "==7.4.0" }, { name = "pyspark", marker = "extra == 'databricks'", specifier = ">=3.5.0" }, { name = "python-dotenv", specifier = ">=1.1.1" }, + { name = "python-dotenv", marker = "extra == 'streamlit'", specifier = "==1.1.1" }, { name = "pyyaml", specifier = ">=6.0.2" }, { name = "qdrant-client", specifier = ">=1.15.1" }, { name = "rich", specifier = ">=14.1.0" }, @@ -1800,11 +1870,13 @@ requires-dist = [ { name = "sqlglot", marker = "extra == 'databricks'", specifier = ">=27.20.0" }, { name = "sqlglot", marker = "extra == 'postgres'", specifier = ">=27.20.0" }, { name = "sqlglot", marker = "extra == 'snowflake'", specifier = ">=27.20.0" }, + { name = "streamlit", marker = "extra == 'streamlit'", specifier = "==1.50.0" }, { name = "symspellpy", specifier = ">=6.9.0" }, { name = "trieregex", specifier = ">=1.0.0" }, { name = "xgboost", specifier = ">=3.0.4" }, + { name = "xlsxwriter", marker = "extra == 'streamlit'", specifier = "==3.2.9" }, ] -provides-extras = ["snowflake", "databricks", "postgres"] +provides-extras = ["snowflake", "databricks", "postgres", "streamlit"] [package.metadata.requires-dev] dev = [ @@ -2908,6 +2980,15 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/79/7b/2c79738432f5c924bef5071f933bcc9efd0473bac3b4aa584a6f7c1c8df8/mypy_extensions-1.1.0-py3-none-any.whl", hash = "sha256:1be4cccdb0f2482337c4743e60421de3a356cd97508abadd57d47403e94f5505", size = 4963, upload-time = "2025-04-22T14:54:22.983Z" }, ] +[[package]] +name = "narwhals" +version = "2.10.2" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/c5/dc/8db74daf8c2690ec696c1d772a33cc01511559ee8a9e92d7ed85a18e3c22/narwhals-2.10.2.tar.gz", hash = "sha256:ff738a08bc993cbb792266bec15346c1d85cc68fdfe82a23283c3713f78bd354", size = 584954, upload-time = "2025-11-04T16:36:42.281Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/47/a9/9e02fa97e421a355fc5e818e9c488080fce04a8e0eebb3ed75a84f041c4a/narwhals-2.10.2-py3-none-any.whl", hash = "sha256:059cd5c6751161b97baedcaf17a514c972af6a70f36a89af17de1a0caf519c43", size = 419573, upload-time = "2025-11-04T16:36:40.574Z" }, +] + [[package]] name = "nest-asyncio" version = "1.6.0" @@ -3481,6 +3562,19 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/40/4b/2028861e724d3bd36227adfa20d3fd24c3fc6d52032f4a93c133be5d17ce/platformdirs-4.4.0-py3-none-any.whl", hash = "sha256:abd01743f24e5287cd7a5db3752faf1a2d65353f38ec26d98e25a6db65958c85", size = 18654, upload-time = "2025-08-26T14:32:02.735Z" }, ] +[[package]] +name = "plotly" +version = "6.4.0" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "narwhals" }, + { name = "packaging" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/06/e6/b768650072837505804bed4790c5449ba348a3b720e27ca7605414e998cd/plotly-6.4.0.tar.gz", hash = "sha256:68c6db2ed2180289ef978f087841148b7efda687552276da15a6e9b92107052a", size = 7012379, upload-time = "2025-11-04T17:59:26.45Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/78/ae/89b45ccccfeebc464c9233de5675990f75241b8ee4cd63227800fdf577d1/plotly-6.4.0-py3-none-any.whl", hash = "sha256:a1062eafbdc657976c2eedd276c90e184ccd6c21282a5e9ee8f20efca9c9a4c5", size = 9892458, upload-time = "2025-11-04T17:59:22.622Z" }, +] + [[package]] name = "pluggy" version = "1.6.0" @@ -3878,6 +3972,20 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/83/d6/887a1ff844e64aa823fb4905978d882a633cfe295c32eacad582b78a7d8b/pydantic_settings-2.11.0-py3-none-any.whl", hash = "sha256:fe2cea3413b9530d10f3a5875adffb17ada5c1e1bab0b2885546d7310415207c", size = 48608, upload-time = "2025-09-24T14:19:10.015Z" }, ] +[[package]] +name = "pydeck" +version = "0.9.1" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "jinja2" }, + { name = "numpy", version = "2.2.6", source = { registry = "https://pypi.org/simple" }, marker = "python_full_version < '3.11'" }, + { name = "numpy", version = "2.3.0", source = { registry = "https://pypi.org/simple" }, marker = "python_full_version >= '3.11'" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/a1/ca/40e14e196864a0f61a92abb14d09b3d3da98f94ccb03b49cf51688140dab/pydeck-0.9.1.tar.gz", hash = "sha256:f74475ae637951d63f2ee58326757f8d4f9cd9f2a457cf42950715003e2cb605", size = 3832240, upload-time = "2024-05-10T15:36:21.153Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/ab/4c/b888e6cf58bd9db9c93f40d1c6be8283ff49d88919231afe93a6bcf61626/pydeck-0.9.1-py2.py3-none-any.whl", hash = "sha256:b3f75ba0d273fc917094fa61224f3f6076ca8752b93d46faf3bcfd9f9d59b038", size = 6900403, upload-time = "2024-05-10T15:36:17.36Z" }, +] + [[package]] name = "pyfakefs" version = "5.9.3" @@ -3918,6 +4026,18 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/61/ad/689f02752eeec26aed679477e80e632ef1b682313be70793d798c1d5fc8f/PyJWT-2.10.1-py3-none-any.whl", hash = "sha256:dcdd193e30abefd5debf142f9adfcdd2b58004e644f25406ffaebd50bd98dacb", size = 22997, upload-time = "2024-11-28T03:43:27.893Z" }, ] +[[package]] +name = "pyngrok" +version = "7.4.0" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "pyyaml" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/a3/40/f06cbabac68bf2ff33b499d32373dedc20ebec3da814c26f82281f3c9bec/pyngrok-7.4.0.tar.gz", hash = "sha256:1f786fed8d156b99bffd713096a7ec8c81181f949c8f5c920f6820e92d6b9876", size = 44530, upload-time = "2025-09-23T14:47:42.642Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/e6/60/00c4570b6de596a87bd05cb828eaf507e6b9ce171ad797e90fe5e8ecbd6e/pyngrok-7.4.0-py3-none-any.whl", hash = "sha256:719ee7bda031c7c3c80e7722aa3103961ac58bfae2167881d57fef28f5c8917a", size = 25434, upload-time = "2025-09-23T14:47:41.229Z" }, +] + [[package]] name = "pyopenssl" version = "25.3.0" @@ -4974,6 +5094,15 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/b7/ce/149a00dd41f10bc29e5921b496af8b574d8413afcd5e30dfa0ed46c2cc5e/six-1.17.0-py2.py3-none-any.whl", hash = "sha256:4721f391ed90541fddacab5acf947aa0d3dc7d27b2e1e8eda2be8970586c3274", size = 11050, upload-time = "2024-12-04T17:35:26.475Z" }, ] +[[package]] +name = "smmap" +version = "5.0.2" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/44/cd/a040c4b3119bbe532e5b0732286f805445375489fceaec1f48306068ee3b/smmap-5.0.2.tar.gz", hash = "sha256:26ea65a03958fa0c8a1c7e8c7a58fdc77221b8910f6be2131affade476898ad5", size = 22329, upload-time = "2025-01-02T07:14:40.909Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/04/be/d09147ad1ec7934636ad912901c5fd7667e1c858e19d355237db0d0cd5e4/smmap-5.0.2-py3-none-any.whl", hash = "sha256:b30115f0def7d7531d22a0fb6502488d879e75b260a9db4d0819cfb25403af5e", size = 24303, upload-time = "2025-01-02T07:14:38.724Z" }, +] + [[package]] name = "sniffio" version = "1.3.1" @@ -5165,6 +5294,36 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/be/72/2db2f49247d0a18b4f1bb9a5a39a0162869acf235f3a96418363947b3d46/starlette-0.48.0-py3-none-any.whl", hash = "sha256:0764ca97b097582558ecb498132ed0c7d942f233f365b86ba37770e026510659", size = 73736, upload-time = "2025-09-13T08:41:03.869Z" }, ] +[[package]] +name = "streamlit" +version = "1.50.0" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "altair" }, + { name = "blinker" }, + { name = "cachetools" }, + { name = "click" }, + { name = "gitpython" }, + { name = "numpy", version = "2.2.6", source = { registry = "https://pypi.org/simple" }, marker = "python_full_version < '3.11'" }, + { name = "numpy", version = "2.3.0", source = { registry = "https://pypi.org/simple" }, marker = "python_full_version >= '3.11'" }, + { name = "packaging" }, + { name = "pandas" }, + { name = "pillow" }, + { name = "protobuf" }, + { name = "pyarrow" }, + { name = "pydeck" }, + { name = "requests" }, + { name = "tenacity" }, + { name = "toml" }, + { name = "tornado" }, + { name = "typing-extensions" }, + { name = "watchdog", marker = "sys_platform != 'darwin'" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/d6/f6/f7d3a0146577c1918439d3163707040f7111a7d2e7e2c73fa7adeb169c06/streamlit-1.50.0.tar.gz", hash = "sha256:87221d568aac585274a05ef18a378b03df332b93e08103fffcf3cd84d852af46", size = 9664808, upload-time = "2025-09-23T19:24:00.31Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/2a/38/991bbf9fa3ed3d9c8e69265fc449bdaade8131c7f0f750dbd388c3c477dc/streamlit-1.50.0-py3-none-any.whl", hash = "sha256:9403b8f94c0a89f80cf679c2fcc803d9a6951e0fba542e7611995de3f67b4bb3", size = 10068477, upload-time = "2025-09-23T19:23:57.245Z" }, +] + [[package]] name = "symspellpy" version = "6.9.0" @@ -5249,6 +5408,15 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/50/79/bcf350609f3a10f09fe4fc207f132085e497fdd3612f3925ab24d86a0ca0/tiktoken-0.11.0-cp313-cp313-win_amd64.whl", hash = "sha256:2177ffda31dec4023356a441793fed82f7af5291120751dee4d696414f54db0c", size = 883901, upload-time = "2025-08-08T23:57:59.359Z" }, ] +[[package]] +name = "toml" +version = "0.10.2" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/be/ba/1f744cdc819428fc6b5084ec34d9b30660f6f9daaf70eead706e3203ec3c/toml-0.10.2.tar.gz", hash = "sha256:b3bda1d108d5dd99f4a20d24d9c348e91c4db7ab1b749200bded2f839ccbe68f", size = 22253, upload-time = "2020-11-01T01:40:22.204Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/44/6f/7120676b6d73228c96e17f1f794d8ab046fc910d781c8d151120c3f1569e/toml-0.10.2-py2.py3-none-any.whl", hash = "sha256:806143ae5bfb6a3c6e736a764057db0e6a0e05e338b5630894a5f779cabb4f9b", size = 16588, upload-time = "2020-11-01T01:40:20.672Z" }, +] + [[package]] name = "tomli" version = "2.2.1" @@ -5499,6 +5667,24 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/63/9a/0962b05b308494e3202d3f794a6e85abe471fe3cafdbcf95c2e8c713aabd/uvloop-0.21.0-cp313-cp313-musllinux_1_2_x86_64.whl", hash = "sha256:a5c39f217ab3c663dc699c04cbd50c13813e31d917642d459fdcec07555cc553", size = 4660018, upload-time = "2024-10-14T23:38:10.888Z" }, ] +[[package]] +name = "watchdog" +version = "6.0.0" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/db/7d/7f3d619e951c88ed75c6037b246ddcf2d322812ee8ea189be89511721d54/watchdog-6.0.0.tar.gz", hash = "sha256:9ddf7c82fda3ae8e24decda1338ede66e1c99883db93711d8fb941eaa2d8c282", size = 131220, upload-time = "2024-11-01T14:07:13.037Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/a9/c7/ca4bf3e518cb57a686b2feb4f55a1892fd9a3dd13f470fca14e00f80ea36/watchdog-6.0.0-py3-none-manylinux2014_aarch64.whl", hash = "sha256:7607498efa04a3542ae3e05e64da8202e58159aa1fa4acddf7678d34a35d4f13", size = 79079, upload-time = "2024-11-01T14:06:59.472Z" }, + { url = "https://files.pythonhosted.org/packages/5c/51/d46dc9332f9a647593c947b4b88e2381c8dfc0942d15b8edc0310fa4abb1/watchdog-6.0.0-py3-none-manylinux2014_armv7l.whl", hash = "sha256:9041567ee8953024c83343288ccc458fd0a2d811d6a0fd68c4c22609e3490379", size = 79078, upload-time = "2024-11-01T14:07:01.431Z" }, + { url = "https://files.pythonhosted.org/packages/d4/57/04edbf5e169cd318d5f07b4766fee38e825d64b6913ca157ca32d1a42267/watchdog-6.0.0-py3-none-manylinux2014_i686.whl", hash = "sha256:82dc3e3143c7e38ec49d61af98d6558288c415eac98486a5c581726e0737c00e", size = 79076, upload-time = "2024-11-01T14:07:02.568Z" }, + { url = "https://files.pythonhosted.org/packages/ab/cc/da8422b300e13cb187d2203f20b9253e91058aaf7db65b74142013478e66/watchdog-6.0.0-py3-none-manylinux2014_ppc64.whl", hash = "sha256:212ac9b8bf1161dc91bd09c048048a95ca3a4c4f5e5d4a7d1b1a7d5752a7f96f", size = 79077, upload-time = "2024-11-01T14:07:03.893Z" }, + { url = "https://files.pythonhosted.org/packages/2c/3b/b8964e04ae1a025c44ba8e4291f86e97fac443bca31de8bd98d3263d2fcf/watchdog-6.0.0-py3-none-manylinux2014_ppc64le.whl", hash = "sha256:e3df4cbb9a450c6d49318f6d14f4bbc80d763fa587ba46ec86f99f9e6876bb26", size = 79078, upload-time = "2024-11-01T14:07:05.189Z" }, + { url = "https://files.pythonhosted.org/packages/62/ae/a696eb424bedff7407801c257d4b1afda455fe40821a2be430e173660e81/watchdog-6.0.0-py3-none-manylinux2014_s390x.whl", hash = "sha256:2cce7cfc2008eb51feb6aab51251fd79b85d9894e98ba847408f662b3395ca3c", size = 79077, upload-time = "2024-11-01T14:07:06.376Z" }, + { url = "https://files.pythonhosted.org/packages/b5/e8/dbf020b4d98251a9860752a094d09a65e1b436ad181faf929983f697048f/watchdog-6.0.0-py3-none-manylinux2014_x86_64.whl", hash = "sha256:20ffe5b202af80ab4266dcd3e91aae72bf2da48c0d33bdb15c66658e685e94e2", size = 79078, upload-time = "2024-11-01T14:07:07.547Z" }, + { url = "https://files.pythonhosted.org/packages/07/f6/d0e5b343768e8bcb4cda79f0f2f55051bf26177ecd5651f84c07567461cf/watchdog-6.0.0-py3-none-win32.whl", hash = "sha256:07df1fdd701c5d4c8e55ef6cf55b8f0120fe1aef7ef39a1c6fc6bc2e606d517a", size = 79065, upload-time = "2024-11-01T14:07:09.525Z" }, + { url = "https://files.pythonhosted.org/packages/db/d9/c495884c6e548fce18a8f40568ff120bc3a4b7b99813081c8ac0c936fa64/watchdog-6.0.0-py3-none-win_amd64.whl", hash = "sha256:cbafb470cf848d93b5d013e2ecb245d4aa1c8fd0504e863ccefa32445359d680", size = 79070, upload-time = "2024-11-01T14:07:10.686Z" }, + { url = "https://files.pythonhosted.org/packages/33/e8/e40370e6d74ddba47f002a32919d91310d6074130fe4e17dabcafc15cbf1/watchdog-6.0.0-py3-none-win_ia64.whl", hash = "sha256:a1914259fa9e1454315171103c6a30961236f508b9b623eae470268bbcc6a22f", size = 79067, upload-time = "2024-11-01T14:07:11.845Z" }, +] + [[package]] name = "watchfiles" version = "1.1.0" @@ -5698,6 +5884,15 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/00/5a/f43bad68b31269a72bdd66102732ea4473e98f421ee9f71379e35dcb56f5/xgboost-3.0.5-py3-none-win_amd64.whl", hash = "sha256:660774249d28a729ba8d22dd3d2c048c56e58f65a683b25ef3252e3383fe956f", size = 56826727, upload-time = "2025-09-05T09:23:55.462Z" }, ] +[[package]] +name = "xlsxwriter" +version = "3.2.9" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/46/2c/c06ef49dc36e7954e55b802a8b231770d286a9758b3d936bd1e04ce5ba88/xlsxwriter-3.2.9.tar.gz", hash = "sha256:254b1c37a368c444eac6e2f867405cc9e461b0ed97a3233b2ac1e574efb4140c", size = 215940, upload-time = "2025-09-16T00:16:21.63Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/3a/0c/3662f4a66880196a590b202f0db82d919dd2f89e99a27fadef91c4a33d41/xlsxwriter-3.2.9-py3-none-any.whl", hash = "sha256:9a5db42bc5dff014806c58a20b9eae7322a134abb6fce3c92c181bfb275ec5b3", size = 175315, upload-time = "2025-09-16T00:16:20.108Z" }, +] + [[package]] name = "xxhash" version = "3.5.0"