diff --git a/.DS_Store b/.DS_Store deleted file mode 100644 index 01222e6..0000000 Binary files a/.DS_Store and /dev/null differ diff --git a/.gitignore b/.gitignore new file mode 100644 index 0000000..87620ac --- /dev/null +++ b/.gitignore @@ -0,0 +1 @@ +.ipynb_checkpoints/ diff --git a/lab-dw-data-structuring-and-combining.ipynb b/lab-dw-data-structuring-and-combining.ipynb index ec4e3f9..235315b 100644 --- a/lab-dw-data-structuring-and-combining.ipynb +++ b/lab-dw-data-structuring-and-combining.ipynb @@ -36,16 +36,984 @@ }, { "cell_type": "code", - "execution_count": null, - "id": "492d06e3-92c7-4105-ac72-536db98d3244", - "metadata": { - "id": "492d06e3-92c7-4105-ac72-536db98d3244" - }, - "outputs": [], + "execution_count": 1, + "id": "c56f21e2-c016-4f75-8324-150fe4ce18a5", + "metadata": {}, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "File 1 shape: (4008, 11)\n", + "File 2 shape: (996, 11)\n", + "File 3 shape: (7070, 11)\n", + "\n", + "File 1 columns:\n", + "['customer', 'state', 'gender', 'education', 'customer_lifetime_value', 'income', 'monthly_premium_auto', 'number_of_open_complaints', 'policy_type', 'vehicle_class', 'total_claim_amount']\n", + "\n", + "File 2 columns:\n", + "['customer', 'state', 'gender', 'education', 'customer_lifetime_value', 'income', 'monthly_premium_auto', 'number_of_open_complaints', 'total_claim_amount', 'policy_type', 'vehicle_class']\n", + "\n", + "File 3 columns:\n", + "['customer', 'state', 'customer_lifetime_value', 'education', 'gender', 'income', 'monthly_premium_auto', 'number_of_open_complaints', 'policy_type', 'total_claim_amount', 'vehicle_class']\n", + "\n", + "Columns missing from File 1:\n", + "[]\n", + "\n", + "Columns missing from File 2:\n", + "[]\n", + "\n", + "Columns missing from File 3:\n", + "[]\n", + "\n", + "Combined dataset shape before cleaning: (12074, 11)\n", + "\n", + "Missing values before cleaning:\n" + ] + }, + { + "data": { + "text/html": [ + "
\n", + "\n", + "\n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + "
missing_values
gender3059
customer_lifetime_value2944
customer2937
state2937
education2937
income2937
monthly_premium_auto2937
number_of_open_complaints2937
policy_type2937
vehicle_class2937
total_claim_amount2937
\n", + "
" + ], + "text/plain": [ + " missing_values\n", + "gender 3059\n", + "customer_lifetime_value 2944\n", + "customer 2937\n", + "state 2937\n", + "education 2937\n", + "income 2937\n", + "monthly_premium_auto 2937\n", + "number_of_open_complaints 2937\n", + "policy_type 2937\n", + "vehicle_class 2937\n", + "total_claim_amount 2937" + ] + }, + "metadata": {}, + "output_type": "display_data" + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "\n", + "Missing values after cleaning:\n" + ] + }, + { + "data": { + "text/html": [ + "
\n", + "\n", + "\n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + "
missing_values
customer0
state0
gender0
education0
customer_lifetime_value0
income0
monthly_premium_auto0
number_of_open_complaints0
policy_type0
vehicle_class0
total_claim_amount0
\n", + "
" + ], + "text/plain": [ + " missing_values\n", + "customer 0\n", + "state 0\n", + "gender 0\n", + "education 0\n", + "customer_lifetime_value 0\n", + "income 0\n", + "monthly_premium_auto 0\n", + "number_of_open_complaints 0\n", + "policy_type 0\n", + "vehicle_class 0\n", + "total_claim_amount 0" + ] + }, + "metadata": {}, + "output_type": "display_data" + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "\n", + "Cleaning results:\n", + "Final shape: (9134, 11)\n", + "Duplicates removed: 3\n", + "Remaining duplicates: 0\n", + "Remaining missing values: 0\n", + "\n", + "Final columns:\n", + "['customer', 'state', 'gender', 'education', 'customer_lifetime_value', 'income', 'monthly_premium_auto', 'number_of_open_complaints', 'policy_type', 'vehicle_class', 'total_claim_amount']\n", + "\n", + "Final data types:\n" + ] + }, + { + "data": { + "text/html": [ + "
\n", + "\n", + "\n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + "
data_type
customerstring
statestring
genderstring
educationstring
customer_lifetime_valueFloat64
incomeint64
monthly_premium_autofloat64
number_of_open_complaintsint64
policy_typestring
vehicle_classstring
total_claim_amountfloat64
\n", + "
" + ], + "text/plain": [ + " data_type\n", + "customer string\n", + "state string\n", + "gender string\n", + "education string\n", + "customer_lifetime_value Float64\n", + "income int64\n", + "monthly_premium_auto float64\n", + "number_of_open_complaints int64\n", + "policy_type string\n", + "vehicle_class string\n", + "total_claim_amount float64" + ] + }, + "metadata": {}, + "output_type": "display_data" + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "\n", + "First 10 cleaned rows:\n" + ] + }, + { + "data": { + "text/html": [ + "
\n", + "\n", + "\n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + "
customerstategendereducationcustomer_lifetime_valueincomemonthly_premium_autonumber_of_open_complaintspolicy_typevehicle_classtotal_claim_amount
0RB50392WashingtonFemaleMaster7714.8801000.00Personal AutoFour-Door Car2.70
1QZ44356ArizonaFemaleBachelor697953.59094.00Personal AutoFour-Door Car1131.46
2AI49188NevadaFemaleBachelor1288743.1748767108.00Personal AutoTwo-Door Car566.47
3WW63253CaliforniaMaleBachelor764586.180106.00Corporate AutoSUV529.88
4GA49547WashingtonMaleHigh School or Below536307.653635768.00Personal AutoFour-Door Car17.27
5OC83172OregonFemaleBachelor825629.786290269.00Personal AutoTwo-Door Car159.38
6XZ87318OregonFemaleCollege538089.865535067.00Corporate AutoFour-Door Car321.60
7CF85061ArizonaMaleMaster721610.030101.00Corporate AutoFour-Door Car363.03
8DY87989OregonMaleBachelor2412750.41407271.00Corporate AutoFour-Door Car511.20
9BQ94931OregonFemaleCollege738817.812881293.00Special AutoFour-Door Car425.53
\n", + "
" + ], + "text/plain": [ + " customer state gender education customer_lifetime_value \\\n", + "0 RB50392 Washington Female Master 7714.88 \n", + "1 QZ44356 Arizona Female Bachelor 697953.59 \n", + "2 AI49188 Nevada Female Bachelor 1288743.17 \n", + "3 WW63253 California Male Bachelor 764586.18 \n", + "4 GA49547 Washington Male High School or Below 536307.65 \n", + "5 OC83172 Oregon Female Bachelor 825629.78 \n", + "6 XZ87318 Oregon Female College 538089.86 \n", + "7 CF85061 Arizona Male Master 721610.03 \n", + "8 DY87989 Oregon Male Bachelor 2412750.4 \n", + "9 BQ94931 Oregon Female College 738817.81 \n", + "\n", + " income monthly_premium_auto number_of_open_complaints policy_type \\\n", + "0 0 1000.0 0 Personal Auto \n", + "1 0 94.0 0 Personal Auto \n", + "2 48767 108.0 0 Personal Auto \n", + "3 0 106.0 0 Corporate Auto \n", + "4 36357 68.0 0 Personal Auto \n", + "5 62902 69.0 0 Personal Auto \n", + "6 55350 67.0 0 Corporate Auto \n", + "7 0 101.0 0 Corporate Auto \n", + "8 14072 71.0 0 Corporate Auto \n", + "9 28812 93.0 0 Special Auto \n", + "\n", + " vehicle_class total_claim_amount \n", + "0 Four-Door Car 2.70 \n", + "1 Four-Door Car 1131.46 \n", + "2 Two-Door Car 566.47 \n", + "3 SUV 529.88 \n", + "4 Four-Door Car 17.27 \n", + "5 Two-Door Car 159.38 \n", + "6 Four-Door Car 321.60 \n", + "7 Four-Door Car 363.03 \n", + "8 Four-Door Car 511.20 \n", + "9 Four-Door Car 425.53 " + ] + }, + "metadata": {}, + "output_type": "display_data" + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "\n", + "Numerical summary:\n" + ] + }, + { + "data": { + "text/html": [ + "
\n", + "\n", + "\n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + "
customer_lifetime_valueincomemonthly_premium_autonumber_of_open_complaintstotal_claim_amount
count9134.09134.009134.009134.009134.00
mean181937.937824.85110.390.38430.48
std440903.8530359.23581.470.91289.62
min1898.010.0061.000.000.10
25%4650.060.0068.000.00266.96
50%7714.8834240.0083.000.00377.50
75%26131.7262446.50109.000.00546.05
max5816655.3599981.0035354.005.002893.24
\n", + "
" + ], + "text/plain": [ + " customer_lifetime_value income monthly_premium_auto \\\n", + "count 9134.0 9134.00 9134.00 \n", + "mean 181937.9 37824.85 110.39 \n", + "std 440903.85 30359.23 581.47 \n", + "min 1898.01 0.00 61.00 \n", + "25% 4650.06 0.00 68.00 \n", + "50% 7714.88 34240.00 83.00 \n", + "75% 26131.72 62446.50 109.00 \n", + "max 5816655.35 99981.00 35354.00 \n", + "\n", + " number_of_open_complaints total_claim_amount \n", + "count 9134.00 9134.00 \n", + "mean 0.38 430.48 \n", + "std 0.91 289.62 \n", + "min 0.00 0.10 \n", + "25% 0.00 266.96 \n", + "50% 0.00 377.50 \n", + "75% 0.00 546.05 \n", + "max 5.00 2893.24 " + ] + }, + "metadata": {}, + "output_type": "display_data" + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "\n", + "Categorical summary:\n" + ] + }, + { + "data": { + "text/html": [ + "
\n", + "\n", + "\n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + "
customerstategendereducationpolicy_typevehicle_class
count913491349134913491349134
unique905652534
topGA49547CaliforniaFemaleBachelorPersonal AutoFour-Door Car
freq231504726274267904640
\n", + "
" + ], + "text/plain": [ + " customer state gender education policy_type vehicle_class\n", + "count 9134 9134 9134 9134 9134 9134\n", + "unique 9056 5 2 5 3 4\n", + "top GA49547 California Female Bachelor Personal Auto Four-Door Car\n", + "freq 2 3150 4726 2742 6790 4640" + ] + }, + "metadata": {}, + "output_type": "display_data" + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "\n", + "Cleaned dataset saved as cleaned_customer_data.csv\n" + ] + } + ], "source": [ - "# Your code goes here" + "import pandas as pd\n", + "import numpy as np\n", + "from IPython.display import display\n", + "\n", + "url1 = \"https://raw.githubusercontent.com/data-bootcamp-v4/data/main/file1.csv\"\n", + "url2 = \"https://raw.githubusercontent.com/data-bootcamp-v4/data/main/file2.csv\"\n", + "url3 = \"https://raw.githubusercontent.com/data-bootcamp-v4/data/main/file3.csv\"\n", + "\n", + "df1 = pd.read_csv(url1)\n", + "df2 = pd.read_csv(url2)\n", + "df3 = pd.read_csv(url3)\n", + "\n", + "def standardize_columns(dataframe):\n", + " dataframe = dataframe.copy()\n", + " dataframe.columns = dataframe.columns.str.strip().str.lower().str.replace(\" \", \"_\", regex=False)\n", + " dataframe = dataframe.rename(columns={\"st\": \"state\"})\n", + " return dataframe\n", + "\n", + "df1 = standardize_columns(df1)\n", + "df2 = standardize_columns(df2)\n", + "df3 = standardize_columns(df3)\n", + "\n", + "print(\"File 1 shape:\", df1.shape)\n", + "print(\"File 2 shape:\", df2.shape)\n", + "print(\"File 3 shape:\", df3.shape)\n", + "\n", + "print(\"\\nFile 1 columns:\")\n", + "print(df1.columns.tolist())\n", + "\n", + "print(\"\\nFile 2 columns:\")\n", + "print(df2.columns.tolist())\n", + "\n", + "print(\"\\nFile 3 columns:\")\n", + "print(df3.columns.tolist())\n", + "\n", + "all_columns = sorted(set(df1.columns) | set(df2.columns) | set(df3.columns))\n", + "\n", + "print(\"\\nColumns missing from File 1:\")\n", + "print(sorted(set(all_columns) - set(df1.columns)))\n", + "\n", + "print(\"\\nColumns missing from File 2:\")\n", + "print(sorted(set(all_columns) - set(df2.columns)))\n", + "\n", + "print(\"\\nColumns missing from File 3:\")\n", + "print(sorted(set(all_columns) - set(df3.columns)))\n", + "\n", + "df = pd.concat([df1, df2, df3], ignore_index=True, sort=False)\n", + "\n", + "print(\"\\nCombined dataset shape before cleaning:\", df.shape)\n", + "\n", + "print(\"\\nMissing values before cleaning:\")\n", + "display(df.isnull().sum().sort_values(ascending=False).to_frame(\"missing_values\"))\n", + "\n", + "df = df.dropna(how=\"all\").reset_index(drop=True)\n", + "\n", + "text_columns = df.select_dtypes(include=[\"object\", \"string\"]).columns\n", + "\n", + "for column in text_columns:\n", + " df[column] = df[column].astype(\"string\").str.strip()\n", + "\n", + "if \"gender\" in df.columns:\n", + " df[\"gender\"] = df[\"gender\"].str.lower().replace({\n", + " \"m\": \"Male\",\n", + " \"male\": \"Male\",\n", + " \"f\": \"Female\",\n", + " \"female\": \"Female\",\n", + " \"femal\": \"Female\"\n", + " })\n", + "\n", + "if \"state\" in df.columns:\n", + " df[\"state\"] = df[\"state\"].replace({\n", + " \"AZ\": \"Arizona\",\n", + " \"Cali\": \"California\",\n", + " \"WA\": \"Washington\"\n", + " })\n", + "\n", + "if \"education\" in df.columns:\n", + " df[\"education\"] = df[\"education\"].replace({\n", + " \"Bachelors\": \"Bachelor\"\n", + " })\n", + "\n", + "if \"vehicle_class\" in df.columns:\n", + " df[\"vehicle_class\"] = df[\"vehicle_class\"].replace({\n", + " \"Sports Car\": \"Luxury\",\n", + " \"Luxury SUV\": \"Luxury\",\n", + " \"Luxury Car\": \"Luxury\"\n", + " })\n", + "\n", + "if \"customer_lifetime_value\" in df.columns:\n", + " df[\"customer_lifetime_value\"] = pd.to_numeric(\n", + " df[\"customer_lifetime_value\"].astype(\"string\").str.replace(\"%\", \"\", regex=False).str.replace(\",\", \"\", regex=False),\n", + " errors=\"coerce\"\n", + " )\n", + "\n", + "def clean_complaints(value):\n", + " if pd.isna(value):\n", + " return np.nan\n", + "\n", + " value = str(value).strip()\n", + "\n", + " if \"/\" in value:\n", + " parts = value.split(\"/\")\n", + " if len(parts) > 1:\n", + " return pd.to_numeric(parts[1], errors=\"coerce\")\n", + "\n", + " return pd.to_numeric(value, errors=\"coerce\")\n", + "\n", + "if \"number_of_open_complaints\" in df.columns:\n", + " df[\"number_of_open_complaints\"] = df[\"number_of_open_complaints\"].apply(clean_complaints)\n", + "\n", + "numeric_columns = [\"income\", \"monthly_premium_auto\", \"total_claim_amount\"]\n", + "\n", + "for column in numeric_columns:\n", + " if column in df.columns:\n", + " df[column] = pd.to_numeric(df[column], errors=\"coerce\")\n", + "\n", + "categorical_columns = df.select_dtypes(include=[\"object\", \"string\"]).columns\n", + "\n", + "for column in categorical_columns:\n", + " mode_value = df[column].mode(dropna=True)\n", + " if not mode_value.empty:\n", + " df[column] = df[column].fillna(mode_value.iloc[0])\n", + "\n", + "numeric_columns_to_fill = [\n", + " \"customer_lifetime_value\",\n", + " \"income\",\n", + " \"monthly_premium_auto\",\n", + " \"number_of_open_complaints\",\n", + " \"total_claim_amount\"\n", + "]\n", + "\n", + "for column in numeric_columns_to_fill:\n", + " if column in df.columns:\n", + " df[column] = df[column].fillna(df[column].median())\n", + "\n", + "duplicates_before = df.duplicated().sum()\n", + "\n", + "df = df.drop_duplicates().reset_index(drop=True)\n", + "\n", + "decimal_columns = [\"customer_lifetime_value\", \"monthly_premium_auto\", \"total_claim_amount\"]\n", + "\n", + "for column in decimal_columns:\n", + " if column in df.columns:\n", + " df[column] = df[column].round(2)\n", + "\n", + "if \"income\" in df.columns:\n", + " df[\"income\"] = df[\"income\"].round().astype(int)\n", + "\n", + "if \"number_of_open_complaints\" in df.columns:\n", + " df[\"number_of_open_complaints\"] = df[\"number_of_open_complaints\"].round().astype(int)\n", + "\n", + "print(\"\\nMissing values after cleaning:\")\n", + "display(df.isnull().sum().sort_values(ascending=False).to_frame(\"missing_values\"))\n", + "\n", + "print(\"\\nCleaning results:\")\n", + "print(\"Final shape:\", df.shape)\n", + "print(\"Duplicates removed:\", duplicates_before)\n", + "print(\"Remaining duplicates:\", df.duplicated().sum())\n", + "print(\"Remaining missing values:\", df.isnull().sum().sum())\n", + "\n", + "print(\"\\nFinal columns:\")\n", + "print(df.columns.tolist())\n", + "\n", + "print(\"\\nFinal data types:\")\n", + "display(df.dtypes.to_frame(\"data_type\"))\n", + "\n", + "print(\"\\nFirst 10 cleaned rows:\")\n", + "display(df.head(10))\n", + "\n", + "print(\"\\nNumerical summary:\")\n", + "display(df.describe().round(2))\n", + "\n", + "print(\"\\nCategorical summary:\")\n", + "display(df.describe(include=[\"object\", \"string\"]))\n", + "\n", + "df.to_csv(\"cleaned_customer_data.csv\", index=False)\n", + "\n", + "print(\"\\nCleaned dataset saved as cleaned_customer_data.csv\")" ] }, + { + "cell_type": "code", + "execution_count": null, + "id": "33075a50-53a1-46ff-ba48-dd656d91fad1", + "metadata": {}, + "outputs": [], + "source": [] + }, { "cell_type": "markdown", "id": "31b8a9e7-7db9-4604-991b-ef6771603e57", @@ -72,14 +1040,571 @@ }, { "cell_type": "code", - "execution_count": null, + "execution_count": 2, "id": "aa10d9b0-1c27-4d3f-a8e4-db6ab73bfd26", "metadata": { "id": "aa10d9b0-1c27-4d3f-a8e4-db6ab73bfd26" }, - "outputs": [], + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Dataset shape: (10910, 27)\n" + ] + }, + { + "data": { + "text/html": [ + "
\n", + "\n", + "\n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + "
unnamed:_0customerstatecustomer_lifetime_valueresponsecoverageeducationeffective_to_dateemploymentstatusgender...number_of_policiespolicy_typepolicyrenew_offer_typesales_channeltotal_claim_amountvehicle_classvehicle_sizevehicle_typemonth
00DK49336Arizona4809.216960NoBasicCollege2011-02-18EmployedM...9Corporate AutoCorporate L3Offer3Agent292.800000Four-Door CarMedsizeA2
11KX64629California2228.525238NoBasicCollege2011-01-18UnemployedF...1Personal AutoPersonal L3Offer4Call Center744.924331Four-Door CarMedsizeA1
22LZ68649Washington14947.917300NoBasicBachelor2011-02-10EmployedM...2Personal AutoPersonal L3Offer3Call Center480.000000SUVMedsizeA2
33XL78013Oregon22332.439460YesExtendedCollege2011-01-11EmployedM...2Corporate AutoCorporate L3Offer2Branch484.013411Four-Door CarMedsizeA1
44QA50777Oregon9025.067525NoPremiumBachelor2011-01-17Medical LeaveF...7Personal AutoPersonal L2Offer1Branch707.925645Four-Door CarMedsizeA1
\n", + "

5 rows × 27 columns

\n", + "
" + ], + "text/plain": [ + " unnamed:_0 customer state customer_lifetime_value response \\\n", + "0 0 DK49336 Arizona 4809.216960 No \n", + "1 1 KX64629 California 2228.525238 No \n", + "2 2 LZ68649 Washington 14947.917300 No \n", + "3 3 XL78013 Oregon 22332.439460 Yes \n", + "4 4 QA50777 Oregon 9025.067525 No \n", + "\n", + " coverage education effective_to_date employmentstatus gender ... \\\n", + "0 Basic College 2011-02-18 Employed M ... \n", + "1 Basic College 2011-01-18 Unemployed F ... \n", + "2 Basic Bachelor 2011-02-10 Employed M ... \n", + "3 Extended College 2011-01-11 Employed M ... \n", + "4 Premium Bachelor 2011-01-17 Medical Leave F ... \n", + "\n", + " number_of_policies policy_type policy renew_offer_type \\\n", + "0 9 Corporate Auto Corporate L3 Offer3 \n", + "1 1 Personal Auto Personal L3 Offer4 \n", + "2 2 Personal Auto Personal L3 Offer3 \n", + "3 2 Corporate Auto Corporate L3 Offer2 \n", + "4 7 Personal Auto Personal L2 Offer1 \n", + "\n", + " sales_channel total_claim_amount vehicle_class vehicle_size \\\n", + "0 Agent 292.800000 Four-Door Car Medsize \n", + "1 Call Center 744.924331 Four-Door Car Medsize \n", + "2 Call Center 480.000000 SUV Medsize \n", + "3 Branch 484.013411 Four-Door Car Medsize \n", + "4 Branch 707.925645 Four-Door Car Medsize \n", + "\n", + " vehicle_type month \n", + "0 A 2 \n", + "1 A 1 \n", + "2 A 2 \n", + "3 A 1 \n", + "4 A 1 \n", + "\n", + "[5 rows x 27 columns]" + ] + }, + "metadata": {}, + "output_type": "display_data" + } + ], + "source": [ + "import pandas as pd\n", + "from IPython.display import display\n", + "\n", + "url = \"https://raw.githubusercontent.com/data-bootcamp-v4/data/main/marketing_customer_analysis_clean.csv\"\n", + "\n", + "df = pd.read_csv(url)\n", + "\n", + "print(\"Dataset shape:\", df.shape)\n", + "\n", + "display(\n", + " df.head()\n", + ")\n", + "\n" + ] + }, + { + "cell_type": "code", + "execution_count": 3, + "id": "48cc4648-de99-424f-ba53-db9c346ad834", + "metadata": {}, + "outputs": [ + { + "data": { + "text/html": [ + "
\n", + "\n", + "\n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + "
sales_channeltotal_revenue
0Agent1810226.82
1Branch1301204.00
2Call Center926600.82
3Web706600.04
\n", + "
" + ], + "text/plain": [ + " sales_channel total_revenue\n", + "0 Agent 1810226.82\n", + "1 Branch 1301204.00\n", + "2 Call Center 926600.82\n", + "3 Web 706600.04" + ] + }, + "metadata": {}, + "output_type": "display_data" + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "The sales channel with the highest total revenue is Agent with 1,810,226.82.\n", + "The sales channel with the lowest total revenue is Web with 706,600.04.\n", + "The revenue difference between the highest and lowest sales channels is 1,103,626.78.\n" + ] + } + ], + "source": [ + "sales_channel_revenue = (\n", + " pd.pivot_table(\n", + " df,\n", + " index=\"sales_channel\",\n", + " values=\"total_claim_amount\",\n", + " aggfunc=\"sum\"\n", + " )\n", + " .rename(\n", + " columns={\n", + " \"total_claim_amount\": \"total_revenue\"\n", + " }\n", + " )\n", + " .sort_values(\n", + " \"total_revenue\",\n", + " ascending=False\n", + " )\n", + " .round(2)\n", + " .reset_index()\n", + ")\n", + "\n", + "display(\n", + " sales_channel_revenue\n", + ")\n", + "\n", + "top_channel = sales_channel_revenue.iloc[0]\n", + "\n", + "lowest_channel = sales_channel_revenue.iloc[-1]\n", + "\n", + "difference = (\n", + " top_channel[\"total_revenue\"]\n", + " - lowest_channel[\"total_revenue\"]\n", + ")\n", + "\n", + "print(\n", + " f\"The sales channel with the highest total revenue is \"\n", + " f\"{top_channel['sales_channel']} with \"\n", + " f\"{top_channel['total_revenue']:,.2f}.\"\n", + ")\n", + "\n", + "print(\n", + " f\"The sales channel with the lowest total revenue is \"\n", + " f\"{lowest_channel['sales_channel']} with \"\n", + " f\"{lowest_channel['total_revenue']:,.2f}.\"\n", + ")\n", + "\n", + "print(\n", + " f\"The revenue difference between the highest and lowest \"\n", + " f\"sales channels is {difference:,.2f}.\"\n", + ")" + ] + }, + { + "cell_type": "code", + "execution_count": 4, + "id": "8a4ddcf2-6001-49a2-b677-207c8f7e624c", + "metadata": {}, + "outputs": [ + { + "data": { + "text/html": [ + "
\n", + "\n", + "\n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + "
genderFM
education
Bachelor7874.277703.60
College7748.828052.46
Doctor7328.517415.33
High School or Below8675.228149.69
Master8157.058168.83
\n", + "
" + ], + "text/plain": [ + "gender F M\n", + "education \n", + "Bachelor 7874.27 7703.60\n", + "College 7748.82 8052.46\n", + "Doctor 7328.51 7415.33\n", + "High School or Below 8675.22 8149.69\n", + "Master 8157.05 8168.83" + ] + }, + "metadata": {}, + "output_type": "display_data" + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "The segment with the highest average customer lifetime value is F customers with High School or Below education at 8,675.22.\n", + "The segment with the lowest average customer lifetime value is F customers with Doctor education at 7,328.51.\n", + "\n", + "Overall average customer lifetime value by gender\n" + ] + }, + { + "data": { + "text/html": [ + "
\n", + "\n", + "\n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + "
average_customer_lifetime_value
gender
F8071.11
M7963.04
\n", + "
" + ], + "text/plain": [ + " average_customer_lifetime_value\n", + "gender \n", + "F 8071.11\n", + "M 7963.04" + ] + }, + "metadata": {}, + "output_type": "display_data" + } + ], "source": [ - "# Your code goes here" + "avg_clv_by_education_gender = pd.pivot_table(\n", + " df,\n", + " index=\"education\",\n", + " columns=\"gender\",\n", + " values=\"customer_lifetime_value\",\n", + " aggfunc=\"mean\"\n", + ").round(2)\n", + "\n", + "display(\n", + " avg_clv_by_education_gender\n", + ")\n", + "\n", + "clv_segments = (\n", + " avg_clv_by_education_gender\n", + " .stack()\n", + " .sort_values(ascending=False)\n", + ")\n", + "\n", + "highest_education = clv_segments.index[0][0]\n", + "highest_gender = clv_segments.index[0][1]\n", + "highest_value = clv_segments.iloc[0]\n", + "\n", + "lowest_education = clv_segments.index[-1][0]\n", + "lowest_gender = clv_segments.index[-1][1]\n", + "lowest_value = clv_segments.iloc[-1]\n", + "\n", + "print(\n", + " f\"The segment with the highest average customer lifetime \"\n", + " f\"value is {highest_gender} customers with \"\n", + " f\"{highest_education} education at {highest_value:,.2f}.\"\n", + ")\n", + "\n", + "print(\n", + " f\"The segment with the lowest average customer lifetime \"\n", + " f\"value is {lowest_gender} customers with \"\n", + " f\"{lowest_education} education at {lowest_value:,.2f}.\"\n", + ")\n", + "\n", + "gender_averages = (\n", + " df.groupby(\"gender\")\n", + " [\"customer_lifetime_value\"]\n", + " .mean()\n", + " .round(2)\n", + " .sort_values(ascending=False)\n", + " .to_frame(\"average_customer_lifetime_value\")\n", + ")\n", + "\n", + "print(\"\\nOverall average customer lifetime value by gender\")\n", + "\n", + "display(\n", + " gender_averages\n", + ")" ] }, { @@ -130,15 +1655,268 @@ }, { "cell_type": "code", - "execution_count": null, + "execution_count": 5, "id": "3a069e0b-b400-470e-904d-d17582191be4", "metadata": { "id": "3a069e0b-b400-470e-904d-d17582191be4" }, - "outputs": [], + "outputs": [ + { + "data": { + "text/html": [ + "
\n", + "\n", + "\n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + "
Policy TypeMonthNumber of Complaints
0Corporate AutoJanuary443
1Corporate AutoFebruary385
2Personal AutoJanuary1728
3Personal AutoFebruary1454
4Special AutoJanuary87
5Special AutoFebruary95
\n", + "
" + ], + "text/plain": [ + " Policy Type Month Number of Complaints\n", + "0 Corporate Auto January 443\n", + "1 Corporate Auto February 385\n", + "2 Personal Auto January 1728\n", + "3 Personal Auto February 1454\n", + "4 Special Auto January 87\n", + "5 Special Auto February 95" + ] + }, + "metadata": {}, + "output_type": "display_data" + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Month with the highest number of complaints for each policy type\n" + ] + }, + { + "data": { + "text/html": [ + "
\n", + "\n", + "\n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + "
Policy TypeMonthNumber of Complaints
0Corporate AutoJanuary443
1Personal AutoJanuary1728
2Special AutoFebruary95
\n", + "
" + ], + "text/plain": [ + " Policy Type Month Number of Complaints\n", + "0 Corporate Auto January 443\n", + "1 Personal Auto January 1728\n", + "2 Special Auto February 95" + ] + }, + "metadata": {}, + "output_type": "display_data" + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "The highest complaint total was recorded for Personal Auto during January, with 1728 complaints.\n" + ] + } + ], "source": [ - "# Your code goes here" + "df[\"effective_to_date\"] = pd.to_datetime(\n", + " df[\"effective_to_date\"],\n", + " errors=\"coerce\"\n", + ")\n", + "\n", + "df[\"number_of_open_complaints\"] = pd.to_numeric(\n", + " df[\"number_of_open_complaints\"],\n", + " errors=\"coerce\"\n", + ").fillna(0)\n", + "\n", + "month_order = [\n", + " \"January\",\n", + " \"February\",\n", + " \"March\",\n", + " \"April\",\n", + " \"May\",\n", + " \"June\",\n", + " \"July\",\n", + " \"August\",\n", + " \"September\",\n", + " \"October\",\n", + " \"November\",\n", + " \"December\"\n", + "]\n", + "\n", + "df[\"month\"] = pd.Categorical(\n", + " df[\"effective_to_date\"].dt.month_name(),\n", + " categories=month_order,\n", + " ordered=True\n", + ")\n", + "\n", + "complaints_long = (\n", + " df.groupby(\n", + " [\"policy_type\", \"month\"],\n", + " observed=True,\n", + " as_index=False\n", + " )\n", + " [\"number_of_open_complaints\"]\n", + " .sum()\n", + " .rename(\n", + " columns={\n", + " \"policy_type\": \"Policy Type\",\n", + " \"month\": \"Month\",\n", + " \"number_of_open_complaints\": \"Number of Complaints\"\n", + " }\n", + " )\n", + " .sort_values(\n", + " [\"Policy Type\", \"Month\"]\n", + " )\n", + " .reset_index(drop=True)\n", + ")\n", + "\n", + "complaints_long[\"Number of Complaints\"] = (\n", + " complaints_long[\"Number of Complaints\"]\n", + " .round()\n", + " .astype(int)\n", + ")\n", + "\n", + "display(\n", + " complaints_long\n", + ")\n", + "\n", + "highest_complaint_rows = (\n", + " complaints_long.loc[\n", + " complaints_long\n", + " .groupby(\"Policy Type\")\n", + " [\"Number of Complaints\"]\n", + " .idxmax()\n", + " ]\n", + " .sort_values(\"Policy Type\")\n", + " .reset_index(drop=True)\n", + ")\n", + "\n", + "print(\n", + " \"Month with the highest number of complaints \"\n", + " \"for each policy type\"\n", + ")\n", + "\n", + "display(\n", + " highest_complaint_rows\n", + ")\n", + "\n", + "overall_highest = complaints_long.loc[\n", + " complaints_long[\"Number of Complaints\"].idxmax()\n", + "]\n", + "\n", + "print(\n", + " f\"The highest complaint total was recorded for \"\n", + " f\"{overall_highest['Policy Type']} during \"\n", + " f\"{overall_highest['Month']}, with \"\n", + " f\"{overall_highest['Number of Complaints']} complaints.\"\n", + ")" ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "af4a9169-da9a-497a-bc22-5f79ac53225f", + "metadata": {}, + "outputs": [], + "source": [] } ], "metadata": { @@ -160,7 +1938,7 @@ "name": "python", "nbconvert_exporter": "python", "pygments_lexer": "ipython3", - "version": "3.9.13" + "version": "3.13.14" } }, "nbformat": 4,