preprocessor.py

"""
Data Preprocessing Module
Handles data cleaning, feature engineering, and preparation for ML
"""

import pandas as pd
import numpy as np
from sklearn.preprocessing import StandardScaler, LabelEncoder
import joblib

class DataPreprocessor:
    def __init__(self):
        self.label_encoders = {}
        self.scaler = StandardScaler()
        
    def clean_data(self, df):
        """
        Clean the data by handling missing values and inconsistencies
        """
        print("🧹 Cleaning data...")
        
        # Make a copy
        df_clean = df.copy()
        
        # Print column names for debugging
        print(f"   Columns found: {list(df_clean.columns)}")
        
        # Standardize column names (remove spaces, convert to title case)
        df_clean.columns = [str(col).strip() for col in df_clean.columns]
        
        # Check for Customer ID column (might have different names)
        customer_id_col = None
        possible_names = ['Customer ID', 'CustomerID', 'customer_id', 'Customer Id', 'customer id', 
                         'CustomerID', 'CUSTOMER_ID', 'Customer_Id']
        
        for col in df_clean.columns:
            if str(col).lower().replace(' ', '').replace('_', '') in ['customerid', 'customer_id']:
                customer_id_col = col
                break
        
        if customer_id_col:
            print(f"   Found Customer ID column: '{customer_id_col}'")
            # Rename to standard 'Customer ID'
            if customer_id_col != 'Customer ID':
                df_clean.rename(columns={customer_id_col: 'Customer ID'}, inplace=True)
        else:
            print("   Warning: No Customer ID column found!")
            # Create a dummy Customer ID if not found
            df_clean['Customer ID'] = range(1, len(df_clean) + 1)
            print("   Created dummy Customer ID column")
        
        # Check for missing values
        missing = df_clean.isnull().sum()
        if missing.sum() > 0:
            print(f"   Found {missing.sum()} missing values")
            # Drop rows with missing values in important columns
            important_cols = ['Customer ID', 'Age', 'Purchase Amount (USD)']
            existing_important = [col for col in important_cols if col in df_clean.columns]
            if existing_important:
                df_clean = df_clean.dropna(subset=existing_important)
                print(f"   Dropped missing values, new shape: {df_clean.shape}")
        
        # Convert data types safely
        if 'Customer ID' in df_clean.columns:
            try:
                df_clean['Customer ID'] = df_clean['Customer ID'].astype(int)
            except Exception as e:
                print(f"   Warning: Could not convert Customer ID to int: {e}")
        
        if 'Age' in df_clean.columns:
            try:
                df_clean['Age'] = df_clean['Age'].astype(int)
            except Exception as e:
                print(f"   Warning: Could not convert Age to int: {e}")
        
        if 'Purchase Amount (USD)' in df_clean.columns:
            try:
                df_clean['Purchase Amount (USD)'] = df_clean['Purchase Amount (USD)'].astype(float)
            except Exception as e:
                print(f"   Warning: Could not convert Purchase Amount to float: {e}")
        
        if 'Review Rating' in df_clean.columns:
            try:
                df_clean['Review Rating'] = df_clean['Review Rating'].astype(float)
            except Exception as e:
                print(f"   Warning: Could not convert Review Rating to float: {e}")
        
        if 'Previous Purchases' in df_clean.columns:
            try:
                df_clean['Previous Purchases'] = df_clean['Previous Purchases'].astype(int)
            except Exception as e:
                print(f"   Warning: Could not convert Previous Purchases to int: {e}")
                if 'Previous Purchases' not in df_clean.columns:
                    df_clean['Previous Purchases'] = 0
        
        # Create age groups if Age column exists
        if 'Age' in df_clean.columns:
            try:
                bins = [0, 25, 35, 50, 100]
                labels = ['Young (18-25)', 'Young Adult (26-35)', 'Adult (36-50)', 'Senior (50+)']
                df_clean['Age_Group'] = pd.cut(df_clean['Age'], bins=bins, labels=labels)
            except Exception as e:
                print(f"   Warning: Could not create Age_Group: {e}")
                df_clean['Age_Group'] = 'Unknown'
        
        print("✅ Data cleaning complete!")
        return df_clean
    
    def create_features(self, df):
        """
        Create features for machine learning
        """
        print("🔧 Creating features...")
        
        df_feat = df.copy()
        
        # Categorical columns to encode (only if they exist)
        cat_cols = ['Gender', 'Category', 'Season', 'Frequency of Purchases', 
                   'Payment Method', 'Shipping Type']
        
        # Encode categorical variables that exist
        for col in cat_cols:
            if col in df_feat.columns:
                try:
                    le = LabelEncoder()
                    df_feat[f'{col}_Encoded'] = le.fit_transform(df_feat[col].astype(str))
                    self.label_encoders[col] = le
                    print(f"   Encoded {col}")
                except Exception as e:
                    print(f"   Warning: Could not encode {col}: {e}")
        
        # Create spending per purchase ratio (if columns exist)
        if 'Purchase Amount (USD)' in df_feat.columns:
            if 'Previous Purchases' in df_feat.columns:
                df_feat['Spending_Ratio'] = df_feat['Purchase Amount (USD)'] / (df_feat['Previous Purchases'] + 1)
            else:
                df_feat['Spending_Ratio'] = df_feat['Purchase Amount (USD)'] / 10
        
        # Create loyalty score
        if 'Previous Purchases' in df_feat.columns:
            max_purchases = df_feat['Previous Purchases'].max()
            if max_purchases > 0:
                df_feat['Loyalty_Score'] = df_feat['Previous Purchases'] / max_purchases
            else:
                df_feat['Loyalty_Score'] = 0
            
            # Add subscription bonus if column exists
            if 'Subscription Status' in df_feat.columns:
                df_feat['Loyalty_Score'] = df_feat['Loyalty_Score'] + (df_feat['Subscription Status'] == 'Yes').astype(int) * 0.5
                df_feat['Loyalty_Score'] = df_feat['Loyalty_Score'].clip(upper=1.0)
        
        print(f"✅ Created {len(df_feat.columns)} features")
        return df_feat
    
    def prepare_ml_features(self, df):
        """
        Prepare features for machine learning (scaling)
        """
        print("📊 Preparing ML features...")
        
        # Select numerical features for ML (only those that exist)
        possible_features = [
            'Age', 
            'Purchase Amount (USD)', 
            'Review Rating', 
            'Previous Purchases',
            'Category_Encoded',
            'Season_Encoded',
            'Frequency of Purchases_Encoded',
            'Spending_Ratio',
            'Loyalty_Score'
        ]
        
        # Filter only available columns
        available_features = [f for f in possible_features if f in df.columns]
        
        if not available_features:
            print("   Warning: No features available for ML!")
            # Create dummy features
            df['dummy_feature'] = 1
            available_features = ['dummy_feature']
        
        # Extract features
        X = df[available_features].values
        
        # Scale the features
        X_scaled = self.scaler.fit_transform(X)
        
        print(f"✅ Prepared {len(available_features)} features for {X.shape[0]} customers")
        print(f"   Features: {available_features}")
        
        return X_scaled, available_features
    
    def save_preprocessor(self, filepath):
        """Save the preprocessor objects"""
        try:
            joblib.dump({
                'label_encoders': self.label_encoders,
                'scaler': self.scaler
            }, filepath)
            print(f"💾 Preprocessor saved to {filepath}")
        except Exception as e:
            print(f"❌ Error saving preprocessor: {e}")
    
    def load_preprocessor(self, filepath):
        """Load the preprocessor objects"""
        try:
            data = joblib.load(filepath)
            self.label_encoders = data['label_encoders']
            self.scaler = data['scaler']
            print(f"📂 Preprocessor loaded from {filepath}")
        except Exception as e:
            print(f"❌ Error loading preprocessor: {e}")

# For testing the module directly
if __name__ == "__main__":
    print("Testing DataPreprocessor class...")
    
    # Create a simple test dataframe
    test_df = pd.DataFrame({
        'Customer ID': [1, 2, 3],
        'Age': [25, 35, 45],
        'Gender': ['Male', 'Female', 'Male'],
        'Purchase Amount (USD)': [100, 200, 150],
        'Category': ['Clothing', 'Footwear', 'Accessories']
    })
    
    # Test the preprocessor
    preprocessor = DataPreprocessor()
    print("✅ DataPreprocessor created successfully")
    
    cleaned_df = preprocessor.clean_data(test_df)
    print("✅ clean_data method works")
    
    featured_df = preprocessor.create_features(cleaned_df)
    print("✅ create_features method works")
    
    X_scaled, features = preprocessor.prepare_ml_features(featured_df)
    print("✅ prepare_ml_features method works")
    
    print(f"\nFinal features: {features}")
    print(f"Scaled data shape: {X_scaled.shape}")