config.yaml

# ── AI Personalization Pipeline Configuration ────────────────────────────────
# Edit this file to customize the persona generation pipeline.
# After changes, re-run notebooks in order: 03 → 04 → 05

# ── Dataset ────────────────────────────────────────────────────────────────────

# Path to the input CSV (or parquet) file for feature engineering.
# Leave empty/null to use the default: data/raw/fraudTrain.csv
# Can also be overridden at runtime with: python run_pipeline.py --data <path>
dataset_path: ~

# ── LLM resilience ───────────────────────────────────────────────────────────
# Max attempts for each LLM call before giving up. Transient API failures
# (HTTP 500/502/503/504, 529 overloaded, 429 rate-limit, connection/timeout)
# are retried with exponential backoff so a momentary blip doesn't kill a long
# pipeline run. Real errors (e.g. 400 bad request, 401 auth) are NOT retried.
llm_max_retries: 5

# ── Data cleaning (column sanitisation at load) ──────────────────────────────
# Wide, sparse exports (e.g. a 150-column "features" CSV where 60+ columns are
# ~99% null and 50+ are constant) otherwise stall the pipeline: NaNs crash the
# scaler/PCA in feature selection, and constant/rank-deficient columns slow the
# VIF gate. The Orchestrator prunes these once, at load, for BOTH the raw-CSV
# and pre-engineered-parquet paths. What was dropped is printed and emitted on
# the bus (Evidence tab) so the decision is fully visible.
data_cleaning:
  enabled: true          # set false to feed the raw table through unchanged
  max_null_frac: 0.5     # drop any column more than this fraction empty (0.5 = >50% null)
  drop_constant: true    # drop zero-variance columns (a single unique non-null value)

# ── Data modality ────────────────────────────────────────────────────────────
# What kind of data is being clustered.
#   auto     — detect automatically (text if a long free-text column is found)
#   tabular  — numeric/categorical table → FeatureEngineer path (default behaviour)
#   text     — articles/documents/reviews → TextPreparer path (embeddings)
# Override at runtime with: python run_pipeline.py --modality text --text-column body
modality: auto

# For text modality: the column holding the documents.
# Leave empty/null to auto-detect the most text-like column.
text_column: ~

# How to embed text into vectors (text modality only).
#   auto         — the agent picks per dataset (short text → TF-IDF, long → transformer)
#   tfidf_svd    — TF-IDF + TruncatedSVD (fast, offline, no extra deps)
#   transformer  — sentence-transformer embeddings (best semantics; needs
#                  `sentence-transformers` installed, else falls back to tfidf_svd)
text_vectorizer: auto

# ── Clustering ────────────────────────────────────────────────────────────────

# Number of customer segments to create.
# Default: 10  |  Recommended range: 4–15
# Fewer clusters → broader, easier-to-action personas
# More clusters  → finer segmentation, more specific personas
n_clusters: ~   # null → auto-select k via silhouette optimizer (recommended)
                # Set an integer (e.g. 6) only to force a specific k

# Clustering algorithm to use.
# Options:
#   auto          — LLM selects based on data shape (recommended)
#   hierarchical  — Agglomerative (Ward linkage). Builds a dendrogram;
#                   good for finding nested/sub-group structure.
#   kmeans        — K-Means. Assigns customers to the nearest centroid.
#                   Fast and interpretable, well-suited for large datasets.
#   dbscan        — Density-based (no k required; auto-detects outliers)
#   gmm           — Gaussian Mixture Model (soft/probabilistic assignments)
#   fuzzy_cmeans  — Fuzzy C-Means (partial memberships)
clustering_algorithm: auto

# AutoML-as-skill candidate tournament.
# When clustering_algorithm/n_clusters are auto, the Clusterer evaluates a
# bounded set of algorithm/k candidates and ranks them by:
#   silhouette + bootstrap stability (ARI) - oversized-cluster penalty.
# This keeps brute-force model selection in deterministic code while agents
# handle routing, interpretation, memory, and business constraints.
enable_automl_candidate_search: true
candidate_search_algorithms: ~       # null → agent defaults per modality
candidate_search_top_n: 8
candidate_stability_repeats: 3
candidate_stability_sample_frac: 0.80

# ── Hierarchical Deepening Loop ───────────────────────────────────────────────
#
# After the initial clustering, any cluster whose share of total customers
# exceeds `max_cluster_size_pct` is automatically split into `sub_n_clusters`
# sub-clusters. This repeats for up to `max_depth` rounds.
#
# IMPORTANT: these three parameters are NO LONGER fixed by config.yaml.
# The Decision Maker (bypass mode) or the user (interactive mode) decides
# them after examining the dataset. The values below are ONLY fallbacks
# if that tuning step fails.
#
#   Bypass mode   → LLM Decision Maker picks values from dataset stats
#                   (n_rows, skewness, modality) and prints reasoning.
#   Interactive   → UI modal asks you to confirm / adjust before clustering.
#
# If you want to force a specific value, say it in your intent text
# (e.g. "max cluster size 25%") — that override takes precedence.
#
# Set max_depth: 0  to disable loop splitting entirely.
# Set max_cluster_size_pct: 1.0  to never trigger (same as max_depth: 0).

max_cluster_size_pct: 0.40   # fallback ONLY — Decision Maker overrides from data
sub_n_clusters: 3            # fallback ONLY — Decision Maker overrides from data
max_depth: 2                 # fallback ONLY — Decision Maker overrides from data

# ── Escalation rules ─────────────────────────────────────────────────────────
# After clustering, if silhouette < `silhouette_target` the pipeline loops back
# to feature selection. After this happens `max_reselect_failures` times in a
# row, the pipeline ESCALATES: it re-runs FeatureEngineer from raw data AND
# asks the Decision Maker to pick a completely different clustering algorithm
# based on the whole failure history.
silhouette_target: 0.5        # starts here; relaxed by -0.1 every max_relax_failures misses
max_reselect_failures: 3      # consecutive misses before re-engineering features
max_relax_failures: 3         # consecutive misses before lowering the silhouette_target

# ── LLM Persona Descriptions ──────────────────────────────────────────────────

# Tone for all Claude-generated persona names, descriptions, and explanations.
# Options:
#   easy          — Plain, everyday language. No jargon. Default.
#   professional  — Formal business language. Suitable for executive presentations.
#   data-driven   — Statistical language; emphasises numbers and ratios throughout.
#   creative      — Vivid metaphors, storytelling, imaginative analogies.
persona_tone: easy

# ── Feature Selection ─────────────────────────────────────────────────────────

# Maximum bottleneck size for the autoencoder used to score feature uniqueness.
# The bottleneck is computed as min(n_features // 5, ae_bottleneck_cap).
# With 379 features: min(75, 32) = 32 → architecture 379→64→32→64→379 (compressing).
# Raise this if you suspect the AE is under-fitting (i.e. reconstruction error
# is uniformly high across all features and scores look meaningless).
ae_bottleneck_cap: 32

# Maximum training iterations for the autoencoder.
# early_stopping=True will exit sooner when validation loss plateaus, so this
# is just an upper bound. Lower values make feature selection faster;
# raise it if the AE convergence warning appears and you want more accuracy.
ae_max_iter: 200

# High-dimensionality guard for the VIF gate.
# The VIF gate is O(rounds × cols × OLS), so a wide DENSE feature matrix (many
# real, varying columns) can make it slow. When the matrix has more columns
# than this, FeatureSelector first keeps only the top-N features by combined
# PCA + autoencoder importance, THEN runs VIF on that bounded set. Real feature
# names are preserved (unlike a PCA projection), so the LLM still selects
# interpretable columns. Most engineered matrices (~60–150 cols) never hit this.
# Set to 0 (or null) to disable the cap.
max_features_for_vif: 150

# ── Classifier ────────────────────────────────────────────────────────────────

# Classifier model for validation.
# Options: auto, random_forest, xgboost, gradient_boosting, logistic_regression
# auto — LLM selects based on dataset characteristics (recommended)
classifier_model: auto

# ── Text / unstructured-data overrides ───────────────────────────────────────
# The defaults above are calibrated for tabular customer-segmentation runs and
# do NOT transfer cleanly to text clustering. Embedding spaces (TF-IDF/SVD or
# sentence transformers) are high-dimensional, cosine-shaped, and produce
# silhouettes that look "low" by Euclidean tabular standards but actually
# encode meaningful topic structure. Likewise, a classifier trained to predict
# cluster membership from short documents will never hit F1=0.70 — that bar
# was set for behavioural feature vectors.
#
# When `state.modality == 'text'`, the orchestrator and agents read these
# overrides instead of the tabular defaults. Set any value to `~` (null) to
# fall back to the tabular default.
text_overrides:
  # Silhouette: cosine silhouette of 0.10–0.30 is typical for usable topic
  # clusters; 0.50 (the tabular default) is essentially unachievable for text.
  silhouette_target: 0.18       # accept clusters that clear ~0.18 cosine sil
  min_silhouette: 0.01          # near-random floor — already used at orchestrator.py:767

  # Classifier F1: distinguishing topic clusters from text is harder than
  # distinguishing customer segments from engineered numeric features. 0.60
  # is the realistic bar (already used at orchestrator.py:768); the per-class
  # F1 floor is also softer because tiny topic clusters lower the macro avg.
  classifier_f1_threshold: 0.60
  per_class_f1_min: 0.30

  # Cluster size: text corpora produce many small natural topics — the 40%
  # tabular cap is too aggressive (would force a single dominant topic to
  # sub-cluster even when it's a coherent group). Loosen to 60%.
  max_cluster_size_pct: 0.60

  # min_cluster_size for text: 2 — already baked into clusterer.py:705 because
  # text has no per-class CV gate forcing a minimum sample size per cluster.
  min_cluster_size: 2

  # Feature-selection gates that don't apply to embeddings. The text path in
  # FeatureSelector already short-circuits PCA/AE/VIF, but the flags below
  # let downstream agents (and the Decision Maker) reason about why.
  skip_vif: true
  skip_correlation: true
  skip_skewness_check: true

  # Algorithm guidance for the Decision Maker. KMeans on L2-normalised
  # embeddings is effectively spherical k-means (cosine-friendly); hierarchical
  # with cosine linkage also works. GMM / DBSCAN / fuzzy_cmeans tend to
  # underperform in high-dim text spaces (Gaussian assumption, distance
  # saturation). algo_recommender.recommend_algorithm reads these to bias
  # its scores when modality='text'.
  preferred_algorithms: [kmeans, hierarchical, nmf, lda]
  discouraged_algorithms: [gmm, dbscan, fuzzy_cmeans]
  # `llm_cluster` is intentionally NOT in preferred_algorithms — it's a
  # rescue path the Decision Maker invokes after other algos exhaust, not a
  # default. Gated at 200 docs in agents/clusterer.py to control cost.