diff --git a/data_structuring_and_combining.ipynb b/data_structuring_and_combining.ipynb new file mode 100644 index 0000000..c3e98d3 --- /dev/null +++ b/data_structuring_and_combining.ipynb @@ -0,0 +1,2413 @@ +{ + "cells": [ + { + "cell_type": "markdown", + "id": "8b63b4ea-9678-473f-873c-78867e2c8e87", + "metadata": {}, + "source": [ + "# Lab: Data Structuring and Combining Data\n", + "\n", + "This notebook completes:\n", + "\n", + "# Challenge 1 — Combining & Cleaning Data \n", + "Using file1.csv, file2.csv, file3.csv\n", + "\n", + "# Challenge 2 — Structuring Data \n", + "Using marketing_customer_analysis_clean.csv\n", + "\n", + "We will:\n", + "\n", + "- Load and clean all datasets \n", + "- Combine them \n", + "- Apply formatting \n", + "- Create pivot tables \n", + "- Reshape data \n", + "- Analyze marketing and customer behavior \n" + ] + }, + { + "cell_type": "code", + "execution_count": 1, + "id": "491798d0-8a33-4146-b296-d04a25b228f0", + "metadata": {}, + "outputs": [], + "source": [ + "# 1. Import core libraries\n", + "import pandas as pd\n", + "import numpy as np\n", + "\n", + "# 2. Display options\n", + "pd.set_option('display.max_columns', None)\n", + "pd.set_option('display.float_format', lambda x: f'{x:,.2f}')\n" + ] + }, + { + "cell_type": "code", + "execution_count": 2, + "id": "435b4d4e-aae6-4562-8583-288ebd5f62e7", + "metadata": {}, + "outputs": [ + { + "data": { + "text/html": [ + "
\n", + "\n", + "\n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + "
customerstategendereducationcustomer_lifetime_valueincomemonthly_premium_autonumber_of_open_complaintspolicy_typevehicle_classtotal_claim_amount
0RB50392WashingtonUnknownMaster793690010000Personal AutoFour-Door Car2.70
1QZ44356ArizonaFBachelor6979530940Personal AutoFour-Door Car1,131.46
2AI49188NevadaFBachelor1288743487671080Personal AutoTwo-Door Car566.47
3WW63253CaliforniaMBachelor76458601060Corporate AutoSUV529.88
4GA49547WashingtonMHigh School or Below53630736357680Personal AutoFour-Door Car17.27
\n", + "
" + ], + "text/plain": [ + " customer state gender education \\\n", + "0 RB50392 Washington Unknown Master \n", + "1 QZ44356 Arizona F Bachelor \n", + "2 AI49188 Nevada F Bachelor \n", + "3 WW63253 California M Bachelor \n", + "4 GA49547 Washington M High School or Below \n", + "\n", + " customer_lifetime_value income monthly_premium_auto \\\n", + "0 793690 0 1000 \n", + "1 697953 0 94 \n", + "2 1288743 48767 108 \n", + "3 764586 0 106 \n", + "4 536307 36357 68 \n", + "\n", + " number_of_open_complaints policy_type vehicle_class \\\n", + "0 0 Personal Auto Four-Door Car \n", + "1 0 Personal Auto Four-Door Car \n", + "2 0 Personal Auto Two-Door Car \n", + "3 0 Corporate Auto SUV \n", + "4 0 Personal Auto Four-Door Car \n", + "\n", + " total_claim_amount \n", + "0 2.70 \n", + "1 1,131.46 \n", + "2 566.47 \n", + "3 529.88 \n", + "4 17.27 " + ] + }, + "execution_count": 2, + "metadata": {}, + "output_type": "execute_result" + } + ], + "source": [ + "# 2. Load the cleaned dataset created in the previous lab\n", + "df_file1 = pd.read_csv(\"cleaned_customer_analysis.csv\")\n", + "\n", + "df_file1.head()" + ] + }, + { + "cell_type": "markdown", + "id": "89df85d9-f06e-48e8-a173-9ff3c33e8d63", + "metadata": {}, + "source": [ + "# Inspecting the cleaned dataset\n", + "\n", + "First check the structure, \n", + "data types,\n", + "and basic statistics to understand what we are working with.\n" + ] + }, + { + "cell_type": "code", + "execution_count": 3, + "id": "e3d0f057-0340-4c53-a2f8-500102214262", + "metadata": {}, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "\n", + "RangeIndex: 1072 entries, 0 to 1071\n", + "Data columns (total 11 columns):\n", + " # Column Non-Null Count Dtype \n", + "--- ------ -------------- ----- \n", + " 0 customer 1071 non-null object \n", + " 1 state 1072 non-null object \n", + " 2 gender 1072 non-null object \n", + " 3 education 1071 non-null object \n", + " 4 customer_lifetime_value 1072 non-null int64 \n", + " 5 income 1072 non-null int64 \n", + " 6 monthly_premium_auto 1072 non-null int64 \n", + " 7 number_of_open_complaints 1072 non-null int64 \n", + " 8 policy_type 1071 non-null object \n", + " 9 vehicle_class 1071 non-null object \n", + " 10 total_claim_amount 1071 non-null float64\n", + "dtypes: float64(1), int64(4), object(6)\n", + "memory usage: 92.3+ KB\n" + ] + }, + { + "data": { + "text/plain": [ + "Index(['customer', 'state', 'gender', 'education', 'customer_lifetime_value',\n", + " 'income', 'monthly_premium_auto', 'number_of_open_complaints',\n", + " 'policy_type', 'vehicle_class', 'total_claim_amount'],\n", + " dtype='object')" + ] + }, + "execution_count": 3, + "metadata": {}, + "output_type": "execute_result" + } + ], + "source": [ + "# 3 — Inspect the Dataset Structure\n", + "\n", + "df_file1.info()\n", + "df_file1.describe()\n", + "df_file1.columns" + ] + }, + { + "cell_type": "code", + "execution_count": 4, + "id": "9d4520a0-d016-401c-a362-2255ca26bb4e", + "metadata": {}, + "outputs": [], + "source": [ + "# Load file2 \n", + "df_file2 = pd.read_csv(\"file2.csv\")\n", + "\n", + "# Load file3\n", + "df_file3 = pd.read_csv(\"file3.csv\")\n" + ] + }, + { + "cell_type": "code", + "execution_count": 5, + "id": "c1b0b485-8ff6-4906-970a-1d118b5ddf53", + "metadata": {}, + "outputs": [ + { + "data": { + "text/html": [ + "
\n", + "\n", + "\n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + "
CustomerSTGENDEREducationCustomer Lifetime ValueIncomeMonthly Premium AutoNumber of Open ComplaintsTotal Claim AmountPolicy TypeVehicle Class
0GS98873ArizonaFBachelor323912.47%16061881/0/00633.60Personal AutoFour-Door Car
1CW49887CaliforniaFMaster462680.11%794871141/0/00547.20Special AutoSUV
2MY31220CaliforniaFCollege899704.02%542301121/0/00537.60Personal AutoTwo-Door Car
3UH35128OregonFCollege2580706.30%712102141/1/001,027.20Personal AutoLuxury Car
4WH52799ArizonaFCollege380812.21%94903941/0/00451.20Corporate AutoTwo-Door Car
\n", + "
" + ], + "text/plain": [ + " Customer ST GENDER Education Customer Lifetime Value Income \\\n", + "0 GS98873 Arizona F Bachelor 323912.47% 16061 \n", + "1 CW49887 California F Master 462680.11% 79487 \n", + "2 MY31220 California F College 899704.02% 54230 \n", + "3 UH35128 Oregon F College 2580706.30% 71210 \n", + "4 WH52799 Arizona F College 380812.21% 94903 \n", + "\n", + " Monthly Premium Auto Number of Open Complaints Total Claim Amount \\\n", + "0 88 1/0/00 633.60 \n", + "1 114 1/0/00 547.20 \n", + "2 112 1/0/00 537.60 \n", + "3 214 1/1/00 1,027.20 \n", + "4 94 1/0/00 451.20 \n", + "\n", + " Policy Type Vehicle Class \n", + "0 Personal Auto Four-Door Car \n", + "1 Special Auto SUV \n", + "2 Personal Auto Two-Door Car \n", + "3 Personal Auto Luxury Car \n", + "4 Corporate Auto Two-Door Car " + ] + }, + "execution_count": 5, + "metadata": {}, + "output_type": "execute_result" + } + ], + "source": [ + "# check file2 to see the dataset\n", + "df_file2.head()" + ] + }, + { + "cell_type": "code", + "execution_count": 6, + "id": "1c1c2dbd-4a74-45b3-bc38-4b3c56a61d83", + "metadata": {}, + "outputs": [ + { + "data": { + "text/html": [ + "
\n", + "\n", + "\n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + "
CustomerStateCustomer Lifetime ValueEducationGenderIncomeMonthly Premium AutoNumber of Open ComplaintsPolicy TypeTotal Claim AmountVehicle Class
0SA25987Washington3,479.14High School or BelowM01040Personal Auto499.20Two-Door Car
1TB86706Arizona2,502.64MasterM0660Personal Auto3.47Two-Door Car
2ZL73902Nevada3,265.16BachelorF25820820Personal Auto393.60Four-Door Car
3KX23516California4,455.84High School or BelowF01210Personal Auto699.62SUV
4FN77294California7,704.96High School or BelowM303661012Personal Auto484.80SUV
\n", + "
" + ], + "text/plain": [ + " Customer State Customer Lifetime Value Education Gender \\\n", + "0 SA25987 Washington 3,479.14 High School or Below M \n", + "1 TB86706 Arizona 2,502.64 Master M \n", + "2 ZL73902 Nevada 3,265.16 Bachelor F \n", + "3 KX23516 California 4,455.84 High School or Below F \n", + "4 FN77294 California 7,704.96 High School or Below M \n", + "\n", + " Income Monthly Premium Auto Number of Open Complaints Policy Type \\\n", + "0 0 104 0 Personal Auto \n", + "1 0 66 0 Personal Auto \n", + "2 25820 82 0 Personal Auto \n", + "3 0 121 0 Personal Auto \n", + "4 30366 101 2 Personal Auto \n", + "\n", + " Total Claim Amount Vehicle Class \n", + "0 499.20 Two-Door Car \n", + "1 3.47 Two-Door Car \n", + "2 393.60 Four-Door Car \n", + "3 699.62 SUV \n", + "4 484.80 SUV " + ] + }, + "execution_count": 6, + "metadata": {}, + "output_type": "execute_result" + } + ], + "source": [ + "# check file3 to see the dataset\n", + "df_file3.head()" + ] + }, + { + "cell_type": "markdown", + "id": "b6868358-abff-41a9-a0fa-010f3e926a1b", + "metadata": {}, + "source": [ + "## Challenge 1 — Cleaning file2 and file3\n", + "\n", + "We apply the same cleaning rules used in the previous lab:\n", + "\n", + "- Standardize column names \n", + "- Fix gender inconsistencies \n", + "- Replace state abbreviations \n", + "- Clean education values \n", + "- Remove % from CLV \n", + "- Fix number_of_open_complaints \n", + "- Convert numeric columns \n" + ] + }, + { + "cell_type": "code", + "execution_count": 7, + "id": "2b173c68-6451-4006-9c1f-279acd1dcdc9", + "metadata": {}, + "outputs": [], + "source": [ + "# Cleaning function reused from previous lab\n", + "def clean_data(df):\n", + " # Standardize column names\n", + " df.columns = df.columns.str.lower().str.replace(\" \", \"_\")\n", + " df = df.rename(columns={\"st\": \"state\"})\n", + " \n", + " # Gender cleanup\n", + " gender_map = {\n", + " \"female\": \"F\", \"Femal\": \"F\", \"f\": \"F\", \"F\": \"F\",\n", + " \"male\": \"M\", \"m\": \"M\", \"Male\": \"M\", \"M\": \"M\"\n", + " }\n", + " df[\"gender\"] = df[\"gender\"].astype(str).map(gender_map).fillna(\"Unknown\")\n", + " \n", + " # State cleanup\n", + " state_map = {\"AZ\": \"Arizona\", \"Cali\": \"California\", \"WA\": \"Washington\"}\n", + " df[\"state\"] = df[\"state\"].replace(state_map)\n", + " \n", + " # Education cleanup\n", + " df[\"education\"] = df[\"education\"].replace(\"Bachelors\", \"Bachelor\")\n", + " \n", + " # CLV cleanup\n", + " df[\"customer_lifetime_value\"] = (\n", + " df[\"customer_lifetime_value\"]\n", + " .astype(str)\n", + " .str.replace(\"%\", \"\")\n", + " )\n", + " df[\"customer_lifetime_value\"] = pd.to_numeric(df[\"customer_lifetime_value\"], errors=\"coerce\")\n", + " \n", + " # Complaints cleanup\n", + " df[\"number_of_open_complaints\"] = (\n", + " df[\"number_of_open_complaints\"]\n", + " .astype(str)\n", + " .str.split(\"/\")\n", + " .str[1]\n", + " )\n", + " df[\"number_of_open_complaints\"] = pd.to_numeric(df[\"number_of_open_complaints\"], errors=\"coerce\")\n", + " \n", + " # Fill nulls\n", + " df[\"gender\"] = df[\"gender\"].fillna(\"Unknown\")\n", + " df[\"state\"] = df[\"state\"].fillna(\"Unknown\")\n", + " df[\"customer_lifetime_value\"] = df[\"customer_lifetime_value\"].fillna(df[\"customer_lifetime_value\"].mean())\n", + " df[\"number_of_open_complaints\"] = df[\"number_of_open_complaints\"].fillna(0)\n", + " df[\"income\"] = df[\"income\"].fillna(0)\n", + " df[\"monthly_premium_auto\"] = df[\"monthly_premium_auto\"].fillna(0)\n", + " \n", + " # Convert numeric columns\n", + " df[\"income\"] = df[\"income\"].astype(int)\n", + " df[\"monthly_premium_auto\"] = df[\"monthly_premium_auto\"].astype(int)\n", + " df[\"number_of_open_complaints\"] = df[\"number_of_open_complaints\"].astype(int)\n", + " df[\"customer_lifetime_value\"] = df[\"customer_lifetime_value\"].astype(int)\n", + " \n", + " # Remove duplicates\n", + " df = df.drop_duplicates().reset_index(drop=True)\n", + " \n", + " return df\n" + ] + }, + { + "cell_type": "code", + "execution_count": 8, + "id": "9d9914cd-8567-4259-a680-eb0e19859647", + "metadata": {}, + "outputs": [ + { + "data": { + "text/html": [ + "
\n", + "\n", + "\n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + "
customerstategendereducationcustomer_lifetime_valueincomemonthly_premium_autonumber_of_open_complaintstotal_claim_amountpolicy_typevehicle_class
0GS98873ArizonaFBachelor32391216061880633.60Personal AutoFour-Door Car
1CW49887CaliforniaFMaster462680794871140547.20Special AutoSUV
2MY31220CaliforniaFCollege899704542301120537.60Personal AutoTwo-Door Car
3UH35128OregonFCollege25807067121021411,027.20Personal AutoLuxury Car
4WH52799ArizonaFCollege38081294903940451.20Corporate AutoTwo-Door Car
\n", + "
" + ], + "text/plain": [ + " customer state gender education customer_lifetime_value income \\\n", + "0 GS98873 Arizona F Bachelor 323912 16061 \n", + "1 CW49887 California F Master 462680 79487 \n", + "2 MY31220 California F College 899704 54230 \n", + "3 UH35128 Oregon F College 2580706 71210 \n", + "4 WH52799 Arizona F College 380812 94903 \n", + "\n", + " monthly_premium_auto number_of_open_complaints total_claim_amount \\\n", + "0 88 0 633.60 \n", + "1 114 0 547.20 \n", + "2 112 0 537.60 \n", + "3 214 1 1,027.20 \n", + "4 94 0 451.20 \n", + "\n", + " policy_type vehicle_class \n", + "0 Personal Auto Four-Door Car \n", + "1 Special Auto SUV \n", + "2 Personal Auto Two-Door Car \n", + "3 Personal Auto Luxury Car \n", + "4 Corporate Auto Two-Door Car " + ] + }, + "execution_count": 8, + "metadata": {}, + "output_type": "execute_result" + } + ], + "source": [ + "# Clean file2 and file3\n", + "clean_file2 = clean_data(df_file2)\n", + "clean_file3 = clean_data(df_file3)\n", + "\n", + "# Display file2\n", + "clean_file2.head()" + ] + }, + { + "cell_type": "code", + "execution_count": 9, + "id": "fc678606-2628-4b0e-a9e4-73e71dd9644d", + "metadata": {}, + "outputs": [ + { + "data": { + "text/html": [ + "
\n", + "\n", + "\n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + "
customerstatecustomer_lifetime_valueeducationgenderincomemonthly_premium_autonumber_of_open_complaintspolicy_typetotal_claim_amountvehicle_class
0SA25987Washington3479High School or BelowM01040Personal Auto499.20Two-Door Car
1TB86706Arizona2502MasterM0660Personal Auto3.47Two-Door Car
2ZL73902Nevada3265BachelorF25820820Personal Auto393.60Four-Door Car
3KX23516California4455High School or BelowF01210Personal Auto699.62SUV
4FN77294California7704High School or BelowM303661010Personal Auto484.80SUV
\n", + "
" + ], + "text/plain": [ + " customer state customer_lifetime_value education gender \\\n", + "0 SA25987 Washington 3479 High School or Below M \n", + "1 TB86706 Arizona 2502 Master M \n", + "2 ZL73902 Nevada 3265 Bachelor F \n", + "3 KX23516 California 4455 High School or Below F \n", + "4 FN77294 California 7704 High School or Below M \n", + "\n", + " income monthly_premium_auto number_of_open_complaints policy_type \\\n", + "0 0 104 0 Personal Auto \n", + "1 0 66 0 Personal Auto \n", + "2 25820 82 0 Personal Auto \n", + "3 0 121 0 Personal Auto \n", + "4 30366 101 0 Personal Auto \n", + "\n", + " total_claim_amount vehicle_class \n", + "0 499.20 Two-Door Car \n", + "1 3.47 Two-Door Car \n", + "2 393.60 Four-Door Car \n", + "3 699.62 SUV \n", + "4 484.80 SUV " + ] + }, + "execution_count": 9, + "metadata": {}, + "output_type": "execute_result" + } + ], + "source": [ + "# Display file3\n", + "clean_file3.head()" + ] + }, + { + "cell_type": "markdown", + "id": "88750dae-fd5c-4bbd-a713-c7ab934983e1", + "metadata": {}, + "source": [ + "# Combine all three cleaned datasets\n", + "\n", + "We now concatenate:\n", + "\n", + "- cleaned file1 \n", + "- cleaned file2 \n", + "- cleaned file3 \n", + "\n", + "into one unified dataset.\n" + ] + }, + { + "cell_type": "code", + "execution_count": 10, + "id": "9cd16b57-ef25-4730-bb3b-9b651518aa63", + "metadata": {}, + "outputs": [ + { + "data": { + "text/html": [ + "
\n", + "\n", + "\n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + "
customerstategendereducationcustomer_lifetime_valueincomemonthly_premium_autonumber_of_open_complaintspolicy_typevehicle_classtotal_claim_amount
0RB50392WashingtonUnknownMaster793690010000Personal AutoFour-Door Car2.70
1QZ44356ArizonaFBachelor6979530940Personal AutoFour-Door Car1,131.46
2AI49188NevadaFBachelor1288743487671080Personal AutoTwo-Door Car566.47
3WW63253CaliforniaMBachelor76458601060Corporate AutoSUV529.88
4GA49547WashingtonMHigh School or Below53630736357680Personal AutoFour-Door Car17.27
\n", + "
" + ], + "text/plain": [ + " customer state gender education \\\n", + "0 RB50392 Washington Unknown Master \n", + "1 QZ44356 Arizona F Bachelor \n", + "2 AI49188 Nevada F Bachelor \n", + "3 WW63253 California M Bachelor \n", + "4 GA49547 Washington M High School or Below \n", + "\n", + " customer_lifetime_value income monthly_premium_auto \\\n", + "0 793690 0 1000 \n", + "1 697953 0 94 \n", + "2 1288743 48767 108 \n", + "3 764586 0 106 \n", + "4 536307 36357 68 \n", + "\n", + " number_of_open_complaints policy_type vehicle_class \\\n", + "0 0 Personal Auto Four-Door Car \n", + "1 0 Personal Auto Four-Door Car \n", + "2 0 Personal Auto Two-Door Car \n", + "3 0 Corporate Auto SUV \n", + "4 0 Personal Auto Four-Door Car \n", + "\n", + " total_claim_amount \n", + "0 2.70 \n", + "1 1,131.46 \n", + "2 566.47 \n", + "3 529.88 \n", + "4 17.27 " + ] + }, + "metadata": {}, + "output_type": "display_data" + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "(9138, 11)\n" + ] + } + ], + "source": [ + "# combine all three dataset\n", + "combined_df = pd.concat([df_file1, clean_file2, clean_file3], axis=0, ignore_index=True)\n", + "\n", + "display(combined_df.head())\n", + "print(combined_df.shape)\n" + ] + }, + { + "cell_type": "markdown", + "id": "7702da3d-5a32-40e1-aee9-d181401ebdd4", + "metadata": {}, + "source": [ + "Check the combined dataset" + ] + }, + { + "cell_type": "code", + "execution_count": 11, + "id": "983e3c29-bb1e-4a82-a778-f2de7b9000b7", + "metadata": {}, + "outputs": [ + { + "data": { + "text/plain": [ + "customer 1\n", + "state 0\n", + "gender 0\n", + "education 1\n", + "customer_lifetime_value 0\n", + "income 0\n", + "monthly_premium_auto 0\n", + "number_of_open_complaints 0\n", + "policy_type 1\n", + "vehicle_class 1\n", + "total_claim_amount 1\n", + "dtype: int64" + ] + }, + "execution_count": 11, + "metadata": {}, + "output_type": "execute_result" + } + ], + "source": [ + "# check for missing values\n", + "combined_df.isna().sum()" + ] + }, + { + "cell_type": "code", + "execution_count": 12, + "id": "709d3e3c-7be4-4f4a-b5f9-7c2ab17780f1", + "metadata": {}, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "\n", + "RangeIndex: 9138 entries, 0 to 9137\n", + "Data columns (total 11 columns):\n", + " # Column Non-Null Count Dtype \n", + "--- ------ -------------- ----- \n", + " 0 customer 9137 non-null object \n", + " 1 state 9138 non-null object \n", + " 2 gender 9138 non-null object \n", + " 3 education 9137 non-null object \n", + " 4 customer_lifetime_value 9138 non-null int64 \n", + " 5 income 9138 non-null int64 \n", + " 6 monthly_premium_auto 9138 non-null int64 \n", + " 7 number_of_open_complaints 9138 non-null int64 \n", + " 8 policy_type 9137 non-null object \n", + " 9 vehicle_class 9137 non-null object \n", + " 10 total_claim_amount 9137 non-null float64\n", + "dtypes: float64(1), int64(4), object(6)\n", + "memory usage: 785.4+ KB\n" + ] + } + ], + "source": [ + "# Inspect Data Types and General info\n", + "combined_df.info()" + ] + }, + { + "cell_type": "code", + "execution_count": 13, + "id": "a14196e1-74bc-42db-8d0b-4c3a8924df74", + "metadata": {}, + "outputs": [ + { + "data": { + "text/plain": [ + "np.int64(3)" + ] + }, + "execution_count": 13, + "metadata": {}, + "output_type": "execute_result" + } + ], + "source": [ + "# check for Duplicate Rows\n", + "\n", + "combined_df.duplicated().sum()" + ] + }, + { + "cell_type": "code", + "execution_count": 14, + "id": "446abb1a-0fbf-4133-81c9-f309eb0e885a", + "metadata": {}, + "outputs": [], + "source": [ + "# Remove duplicates\n", + "combined_df = combined_df.drop_duplicates()\n", + "\n", + "combined_df.reset_index(drop=True, inplace=True)" + ] + }, + { + "cell_type": "code", + "execution_count": 15, + "id": "c74ffabb-2b91-467f-9d71-b9b8870d709c", + "metadata": {}, + "outputs": [ + { + "data": { + "text/plain": [ + "" + ] + }, + "execution_count": 15, + "metadata": {}, + "output_type": "execute_result" + } + ], + "source": [ + "combined_df.duplicated" + ] + }, + { + "cell_type": "code", + "execution_count": 16, + "id": "276bdbf9-1bf5-420f-897f-52f0ee9be97a", + "metadata": {}, + "outputs": [ + { + "data": { + "text/html": [ + "
\n", + "\n", + "\n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + "
customer_lifetime_valueincomemonthly_premium_autonumber_of_open_complaintstotal_claim_amount
count9,135.009,135.009,135.009,135.009,134.00
mean182,594.1437,820.71110.380.04430.48
std441,207.5130,360.15581.440.31289.62
min1,898.000.000.000.000.10
25%4,650.000.0068.000.00266.96
50%7,716.0034,236.0083.000.00377.51
75%26,199.5062,446.00109.000.00546.05
max5,816,655.0099,981.0035,354.005.002,893.24
\n", + "
" + ], + "text/plain": [ + " customer_lifetime_value income monthly_premium_auto \\\n", + "count 9,135.00 9,135.00 9,135.00 \n", + "mean 182,594.14 37,820.71 110.38 \n", + "std 441,207.51 30,360.15 581.44 \n", + "min 1,898.00 0.00 0.00 \n", + "25% 4,650.00 0.00 68.00 \n", + "50% 7,716.00 34,236.00 83.00 \n", + "75% 26,199.50 62,446.00 109.00 \n", + "max 5,816,655.00 99,981.00 35,354.00 \n", + "\n", + " number_of_open_complaints total_claim_amount \n", + "count 9,135.00 9,134.00 \n", + "mean 0.04 430.48 \n", + "std 0.31 289.62 \n", + "min 0.00 0.10 \n", + "25% 0.00 266.96 \n", + "50% 0.00 377.51 \n", + "75% 0.00 546.05 \n", + "max 5.00 2,893.24 " + ] + }, + "execution_count": 16, + "metadata": {}, + "output_type": "execute_result" + } + ], + "source": [ + "# statistical Summary\n", + "combined_df.describe()" + ] + }, + { + "cell_type": "code", + "execution_count": 17, + "id": "969c9dcc-cd49-40b2-babf-2ae6d9ed4a4c", + "metadata": {}, + "outputs": [], + "source": [ + "# Save the final dataset\n", + "combined_df.to_csv(\n", + " \"combined_cleaned_insurance.csv\",\n", + " index=False\n", + ")" + ] + }, + { + "cell_type": "markdown", + "id": "3876a902-9c82-4af5-8970-727ce8e02913", + "metadata": {}, + "source": [ + "CONCLUSION:\n", + "\n", + "We first reused the cleaning function developed in the previous lab to ensure consistency across all datasets. \n", + "\n", + "After cleaning file2 and file3, we concatenated the three datasets because they share the same structure and represent additional customer records.\n", + "* we did not use merge because all 3 files contain the same variables, and we adding more customers(row not new columns.)\n", + "* \n", + "Finally, duplicate records were removed and the combined dataset was saved for further analysis." + ] + }, + { + "cell_type": "markdown", + "id": "728884bb-af95-43a7-970f-7a94389da668", + "metadata": {}, + "source": [ + "# Challenge 2 — Structuring Data\n", + "\n", + "We now load the marketing dataset:\n", + "\n", + "`marketing_customer_analysis_clean.csv`\n", + "\n", + "This dataset includes:\n", + "\n", + "- Sales channel \n", + "- Customer lifetime value \n", + "- Response \n", + "- Policy details \n", + "- Vehicle details \n" + ] + }, + { + "cell_type": "code", + "execution_count": 18, + "id": "9f7f8ff3-cc82-476f-bcac-dbefad6ab2c6", + "metadata": {}, + "outputs": [ + { + "data": { + "text/html": [ + "
\n", + "\n", + "\n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + "
unnamed:_0customerstatecustomer_lifetime_valueresponsecoverageeducationeffective_to_dateemploymentstatusgenderincomelocation_codemarital_statusmonthly_premium_automonths_since_last_claimmonths_since_policy_inceptionnumber_of_open_complaintsnumber_of_policiespolicy_typepolicyrenew_offer_typesales_channeltotal_claim_amountvehicle_classvehicle_sizevehicle_typemonth
00DK49336Arizona4,809.22NoBasicCollege2011-02-18EmployedM48029SuburbanMarried617.00520.009Corporate AutoCorporate L3Offer3Agent292.80Four-Door CarMedsizeA2
11KX64629California2,228.53NoBasicCollege2011-01-18UnemployedF0SuburbanSingle643.00260.001Personal AutoPersonal L3Offer4Call Center744.92Four-Door CarMedsizeA1
22LZ68649Washington14,947.92NoBasicBachelor2011-02-10EmployedM22139SuburbanSingle10034.00310.002Personal AutoPersonal L3Offer3Call Center480.00SUVMedsizeA2
33XL78013Oregon22,332.44YesExtendedCollege2011-01-11EmployedM49078SuburbanSingle9710.0030.002Corporate AutoCorporate L3Offer2Branch484.01Four-Door CarMedsizeA1
44QA50777Oregon9,025.07NoPremiumBachelor2011-01-17Medical LeaveF23675SuburbanMarried11715.15310.387Personal AutoPersonal L2Offer1Branch707.93Four-Door CarMedsizeA1
\n", + "
" + ], + "text/plain": [ + " unnamed:_0 customer state customer_lifetime_value response \\\n", + "0 0 DK49336 Arizona 4,809.22 No \n", + "1 1 KX64629 California 2,228.53 No \n", + "2 2 LZ68649 Washington 14,947.92 No \n", + "3 3 XL78013 Oregon 22,332.44 Yes \n", + "4 4 QA50777 Oregon 9,025.07 No \n", + "\n", + " coverage education effective_to_date employmentstatus gender income \\\n", + "0 Basic College 2011-02-18 Employed M 48029 \n", + "1 Basic College 2011-01-18 Unemployed F 0 \n", + "2 Basic Bachelor 2011-02-10 Employed M 22139 \n", + "3 Extended College 2011-01-11 Employed M 49078 \n", + "4 Premium Bachelor 2011-01-17 Medical Leave F 23675 \n", + "\n", + " location_code marital_status monthly_premium_auto months_since_last_claim \\\n", + "0 Suburban Married 61 7.00 \n", + "1 Suburban Single 64 3.00 \n", + "2 Suburban Single 100 34.00 \n", + "3 Suburban Single 97 10.00 \n", + "4 Suburban Married 117 15.15 \n", + "\n", + " months_since_policy_inception number_of_open_complaints \\\n", + "0 52 0.00 \n", + "1 26 0.00 \n", + "2 31 0.00 \n", + "3 3 0.00 \n", + "4 31 0.38 \n", + "\n", + " number_of_policies policy_type policy renew_offer_type \\\n", + "0 9 Corporate Auto Corporate L3 Offer3 \n", + "1 1 Personal Auto Personal L3 Offer4 \n", + "2 2 Personal Auto Personal L3 Offer3 \n", + "3 2 Corporate Auto Corporate L3 Offer2 \n", + "4 7 Personal Auto Personal L2 Offer1 \n", + "\n", + " sales_channel total_claim_amount vehicle_class vehicle_size vehicle_type \\\n", + "0 Agent 292.80 Four-Door Car Medsize A \n", + "1 Call Center 744.92 Four-Door Car Medsize A \n", + "2 Call Center 480.00 SUV Medsize A \n", + "3 Branch 484.01 Four-Door Car Medsize A \n", + "4 Branch 707.93 Four-Door Car Medsize A \n", + "\n", + " month \n", + "0 2 \n", + "1 1 \n", + "2 2 \n", + "3 1 \n", + "4 1 " + ] + }, + "execution_count": 18, + "metadata": {}, + "output_type": "execute_result" + } + ], + "source": [ + "# Load the cleaned marketing dataset\n", + "\n", + "marketing = pd.read_csv(\"marketing_customer_analysis_clean.csv\")\n", + "\n", + "#view the first five rows\n", + "marketing.head()\n" + ] + }, + { + "cell_type": "code", + "execution_count": 19, + "id": "4de9c113-de58-4643-bf88-f24fca12728d", + "metadata": {}, + "outputs": [ + { + "data": { + "text/plain": [ + "(10910, 27)" + ] + }, + "execution_count": 19, + "metadata": {}, + "output_type": "execute_result" + } + ], + "source": [ + "# Check the dimensions\n", + "marketing.shape" + ] + }, + { + "cell_type": "code", + "execution_count": 20, + "id": "ce48e197-b04c-4877-9561-9a2873751353", + "metadata": {}, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "\n", + "RangeIndex: 10910 entries, 0 to 10909\n", + "Data columns (total 27 columns):\n", + " # Column Non-Null Count Dtype \n", + "--- ------ -------------- ----- \n", + " 0 unnamed:_0 10910 non-null int64 \n", + " 1 customer 10910 non-null object \n", + " 2 state 10910 non-null object \n", + " 3 customer_lifetime_value 10910 non-null float64\n", + " 4 response 10910 non-null object \n", + " 5 coverage 10910 non-null object \n", + " 6 education 10910 non-null object \n", + " 7 effective_to_date 10910 non-null object \n", + " 8 employmentstatus 10910 non-null object \n", + " 9 gender 10910 non-null object \n", + " 10 income 10910 non-null int64 \n", + " 11 location_code 10910 non-null object \n", + " 12 marital_status 10910 non-null object \n", + " 13 monthly_premium_auto 10910 non-null int64 \n", + " 14 months_since_last_claim 10910 non-null float64\n", + " 15 months_since_policy_inception 10910 non-null int64 \n", + " 16 number_of_open_complaints 10910 non-null float64\n", + " 17 number_of_policies 10910 non-null int64 \n", + " 18 policy_type 10910 non-null object \n", + " 19 policy 10910 non-null object \n", + " 20 renew_offer_type 10910 non-null object \n", + " 21 sales_channel 10910 non-null object \n", + " 22 total_claim_amount 10910 non-null float64\n", + " 23 vehicle_class 10910 non-null object \n", + " 24 vehicle_size 10910 non-null object \n", + " 25 vehicle_type 10910 non-null object \n", + " 26 month 10910 non-null int64 \n", + "dtypes: float64(4), int64(6), object(17)\n", + "memory usage: 2.2+ MB\n" + ] + } + ], + "source": [ + "# Check the column names and data types\n", + "marketing.info()" + ] + }, + { + "cell_type": "code", + "execution_count": 21, + "id": "a43eff28-f28e-44f2-b8ac-67da62154228", + "metadata": {}, + "outputs": [ + { + "data": { + "text/html": [ + "
\n", + "\n", + "\n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + "
unnamed:_0customer_lifetime_valueincomemonthly_premium_automonths_since_last_claimmonths_since_policy_inceptionnumber_of_open_complaintsnumber_of_policiestotal_claim_amountmonth
count10,910.0010,910.0010,910.0010,910.0010,910.0010,910.0010,910.0010,910.0010,910.0010,910.00
mean5,454.508,018.2437,536.2893.2015.1548.090.382.98434.891.47
std3,149.596,885.0830,359.2034.449.7827.940.892.40292.180.50
min0.001,898.010.0061.000.000.000.001.000.101.00
25%2,727.254,014.450.0068.007.0024.000.001.00271.081.00
50%5,454.505,771.1533,813.5083.0015.0048.000.002.00382.561.00
75%8,181.758,992.7862,250.75109.0023.0071.000.384.00547.202.00
max10,909.0083,325.3899,981.00298.0035.0099.005.009.002,893.242.00
\n", + "
" + ], + "text/plain": [ + " unnamed:_0 customer_lifetime_value income monthly_premium_auto \\\n", + "count 10,910.00 10,910.00 10,910.00 10,910.00 \n", + "mean 5,454.50 8,018.24 37,536.28 93.20 \n", + "std 3,149.59 6,885.08 30,359.20 34.44 \n", + "min 0.00 1,898.01 0.00 61.00 \n", + "25% 2,727.25 4,014.45 0.00 68.00 \n", + "50% 5,454.50 5,771.15 33,813.50 83.00 \n", + "75% 8,181.75 8,992.78 62,250.75 109.00 \n", + "max 10,909.00 83,325.38 99,981.00 298.00 \n", + "\n", + " months_since_last_claim months_since_policy_inception \\\n", + "count 10,910.00 10,910.00 \n", + "mean 15.15 48.09 \n", + "std 9.78 27.94 \n", + "min 0.00 0.00 \n", + "25% 7.00 24.00 \n", + "50% 15.00 48.00 \n", + "75% 23.00 71.00 \n", + "max 35.00 99.00 \n", + "\n", + " number_of_open_complaints number_of_policies total_claim_amount \\\n", + "count 10,910.00 10,910.00 10,910.00 \n", + "mean 0.38 2.98 434.89 \n", + "std 0.89 2.40 292.18 \n", + "min 0.00 1.00 0.10 \n", + "25% 0.00 1.00 271.08 \n", + "50% 0.00 2.00 382.56 \n", + "75% 0.38 4.00 547.20 \n", + "max 5.00 9.00 2,893.24 \n", + "\n", + " month \n", + "count 10,910.00 \n", + "mean 1.47 \n", + "std 0.50 \n", + "min 1.00 \n", + "25% 1.00 \n", + "50% 1.00 \n", + "75% 2.00 \n", + "max 2.00 " + ] + }, + "execution_count": 21, + "metadata": {}, + "output_type": "execute_result" + } + ], + "source": [ + "# Display summary statistics\n", + "\n", + "marketing.describe()" + ] + }, + { + "cell_type": "markdown", + "id": "8ced82ad-73e6-4160-b3b4-d4bd71efa334", + "metadata": {}, + "source": [ + "# Pivot 1 — Total revenue by sales channel\n", + "\n", + "We compute:\n", + "\n", + "*Total Revenue = Monthly Premium Auto × Number of Policies**\n", + "\n", + "Then pivot by sales channel.\n" + ] + }, + { + "cell_type": "code", + "execution_count": 22, + "id": "f88f38f2-86d7-4f45-aa67-d60df6f261bc", + "metadata": {}, + "outputs": [ + { + "data": { + "text/html": [ + "
\n", + "\n", + "\n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + "
total_revenue
sales_channel
Agent1130616
Branch818951
Call Center589902
Web471197
\n", + "
" + ], + "text/plain": [ + " total_revenue\n", + "sales_channel \n", + "Agent 1130616\n", + "Branch 818951\n", + "Call Center 589902\n", + "Web 471197" + ] + }, + "execution_count": 22, + "metadata": {}, + "output_type": "execute_result" + } + ], + "source": [ + "# Create a pivot table showing total revenue by sales channel\n", + "marketing[\"total_revenue\"] = (\n", + " marketing[\"monthly_premium_auto\"] * marketing[\"number_of_policies\"]\n", + ")\n", + "\n", + "pivot_revenue = pd.pivot_table(\n", + " marketing,\n", + " values=\"total_revenue\",\n", + " index=\"sales_channel\",\n", + " aggfunc=\"sum\"\n", + ").round(2)\n", + "\n", + "pivot_revenue\n" + ] + }, + { + "cell_type": "code", + "execution_count": 23, + "id": "09dcfd13-85b9-474b-8074-3b6dc13d7c32", + "metadata": {}, + "outputs": [ + { + "data": { + "text/html": [ + "
\n", + "\n", + "\n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + "
genderFM
education
Bachelor7,874.277,703.60
College7,748.828,052.46
Doctor7,328.517,415.33
High School or Below8,675.228,149.69
Master8,157.058,168.83
\n", + "
" + ], + "text/plain": [ + "gender F M\n", + "education \n", + "Bachelor 7,874.27 7,703.60\n", + "College 7,748.82 8,052.46\n", + "Doctor 7,328.51 7,415.33\n", + "High School or Below 8,675.22 8,149.69\n", + "Master 8,157.05 8,168.83" + ] + }, + "execution_count": 23, + "metadata": {}, + "output_type": "execute_result" + } + ], + "source": [ + "# EXERCISE 2:\n", + "# Average Customer Lifetime Value by gender and education\n", + "# Round values to two decimal places\n", + "clv_table = pd.pivot_table(\n", + " marketing,\n", + " index=\"education\",\n", + " columns=\"gender\",\n", + " values=\"customer_lifetime_value\",\n", + " aggfunc=\"mean\"\n", + ")\n", + "\n", + "# Round to two decimal places\n", + "clv_table = clv_table.round(2)\n", + "\n", + "# Display the table\n", + "clv_table" + ] + }, + { + "cell_type": "markdown", + "id": "5ce48dac-8147-4fa3-88c0-bdfe3604d1bf", + "metadata": {}, + "source": [ + "## Bonus — Complaints by month and policy type (long format)\n", + "\n", + "We extract month from `effective_to_date` and count complaints.\n" + ] + }, + { + "cell_type": "code", + "execution_count": 24, + "id": "641b5864-484e-4eea-88af-9171fbbab21b", + "metadata": {}, + "outputs": [ + { + "data": { + "text/html": [ + "
\n", + "\n", + "\n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + "
policy_typemonthnumber_of_open_complaints
0Corporate AutoFebruary385.21
1Corporate AutoJanuary443.43
2Personal AutoFebruary1,453.68
3Personal AutoJanuary1,727.61
4Special AutoFebruary95.23
\n", + "
" + ], + "text/plain": [ + " policy_type month number_of_open_complaints\n", + "0 Corporate Auto February 385.21\n", + "1 Corporate Auto January 443.43\n", + "2 Personal Auto February 1,453.68\n", + "3 Personal Auto January 1,727.61\n", + "4 Special Auto February 95.23" + ] + }, + "execution_count": 24, + "metadata": {}, + "output_type": "execute_result" + } + ], + "source": [ + "# Count complaints by policy type and month\n", + "\n", + "marketing[\"effective_to_date\"] = pd.to_datetime(marketing[\"effective_to_date\"])\n", + "marketing[\"month\"] = marketing[\"effective_to_date\"].dt.month_name()\n", + "\n", + "complaints_long = marketing.groupby(\n", + " [\"policy_type\", \"month\"]\n", + ")[\"number_of_open_complaints\"].sum().reset_index()\n", + "\n", + "complaints_long.head()\n" + ] + }, + { + "cell_type": "code", + "execution_count": 25, + "id": "9874ea6b-8401-4088-b6b8-be869a90e9e6", + "metadata": {}, + "outputs": [ + { + "ename": "SyntaxError", + "evalue": "invalid syntax (3568774380.py, line 3)", + "output_type": "error", + "traceback": [ + " \u001b[36mCell\u001b[39m\u001b[36m \u001b[39m\u001b[32mIn[25]\u001b[39m\u001b[32m, line 3\u001b[39m\n\u001b[31m \u001b[39m\u001b[31mIn this notebook, we:\u001b[39m\n ^\n\u001b[31mSyntaxError\u001b[39m\u001b[31m:\u001b[39m invalid syntax\n" + ] + } + ], + "source": [ + "# Conclusion\n", + "\n", + "In this notebook, we:\n", + "\n", + "### ✔ Cleaned file2 and file3 \n", + "### ✔ Combined all three datasets \n", + "### ✔ Loaded the marketing dataset \n", + "### ✔ Created pivot tables \n", + "### ✔ Generated long-format complaint tables \n", + "### ✔ Completed both Challenge 1 and Challenge 2 \n", + "\n", + "This notebook is ready for submission.\n" + ] + } + ], + "metadata": { + "kernelspec": { + "display_name": "anaconda-2025.12-py3.13", + "language": "python", + "name": "conda-env-anaconda-2025.12-py3.13" + }, + "language_info": { + "codemirror_mode": { + "name": "ipython", + "version": 3 + }, + "file_extension": ".py", + "mimetype": "text/x-python", + "name": "python", + "nbconvert_exporter": "python", + "pygments_lexer": "ipython3", + "version": "3.13.9" + } + }, + "nbformat": 4, + "nbformat_minor": 5 +}