Machine Learning with Spark.ipynb

{
 "cells": [
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "[Spark with Jupyter Notebook on MacOS (2.0.0 and higher)](https://medium.com/@roshinijohri/spark-with-jupyter-notebook-on-macos-2-0-0-and-higher-c61b971b5007)\n",
    "==========================================================================================\n",
    "\n",
    "#### Run in Terminal:\n",
    "$\\textrm{brew install apache-spark}$\n",
    "\n",
    "$\\textrm{brew info apache-spark}$\n",
    "\n",
    "$\\textrm{export SPARK_HOME='/usr/local/Cellar/apache-spark/2.4.5/libexec/'}$ -> Edit depending on version\n",
    "\n",
    "$\\textrm{pyspark}$"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Welcome to\n",
      "      ____              __\n",
      "     / __/__  ___ _____/ /__\n",
      "    _\\ \\/ _ \\/ _ `/ __/  '_/\n",
      "   /__ / .__/\\_,_/_/ /_/\\_\\   version 2.4.5\n",
      "      /_/\n",
      "\n",
      "Using Python version 3.7.4 (default, Aug 13 2019 15:17:50)\n",
      "SparkSession available as 'spark'.\n"
     ]
    }
   ],
   "source": [
    "import os\n",
    "exec(open(os.path.join(os.environ['SPARK_HOME'], 'python/pyspark/shell.py')).read())"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [],
   "source": [
    "import pyspark\n",
    "from pyspark.sql.session import SparkSession\n",
    "spark = SparkSession.builder \\\n",
    "    .appName('spark test') \\\n",
    "    .getOrCreate() \\\n",
    "\n",
    "columns = ['id', 'dogs', 'cats']\n",
    "vals = [\n",
    "    (1, 2, 0),\n",
    "    (2, 0, 1)\n",
    "]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "+---+----+----+\n",
      "| id|dogs|cats|\n",
      "+---+----+----+\n",
      "|  1|   2|   0|\n",
      "|  2|   0|   1|\n",
      "+---+----+----+\n",
      "\n"
     ]
    }
   ],
   "source": [
    "# Create DataFrame\n",
    "df = spark.createDataFrame(vals, columns)\n",
    "df.show()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Numeric Feature Extraction"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [],
   "source": [
    "import pyspark\n",
    "from pyspark import SparkConf\n",
    "from pyspark.sql import SparkSession\n",
    "from pyspark.ml.feature import RegexTokenizer, VectorAssembler, Normalizer, StandardScaler, MinMaxScaler, MaxAbsScaler, \\\n",
    "                               CountVectorizer, IDF, StringIndexer, PCA, StopWordsRemover\n",
    "from pyspark.ml import Pipeline\n",
    "from pyspark.ml.regression import LinearRegression\n",
    "from pyspark.ml.classification import LogisticRegression\n",
    "from pyspark.ml.evaluation import MulticlassClassificationEvaluator\n",
    "from pyspark.ml.clustering import KMeans\n",
    "from pyspark.ml.tuning import CrossValidator, ParamGridBuilder\n",
    "from pyspark.sql.functions import avg, col, concat, count, desc, explode, lit, min, max, split, stddev, udf\n",
    "from pyspark.sql.types import IntegerType\n",
    "\n",
    "import re"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Instantiate a Spark session \n",
    "# The entry point to programming Spark with the Dataset and DataFrame API\n",
    "# Note: master(): sets the Spark master URL to connect to, such as \"local\" to run locally, \"local[4]\" to run \n",
    "#                 locally with 4 cores, or \"spark://master:7077\" to run on a Spark standalone cluster\n",
    "#       appName(): sets a name for the application, which will be shown in the Spark web UI\n",
    "#       getOrCreate(): get or instantiate a SparkContext and register it as a singleton object\n",
    "spark = SparkSession.builder \\\n",
    "    .master('local') \\\n",
    "    .appName('Word Count') \\\n",
    "    .getOrCreate()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "[('spark.master', 'local'),\n",
       " ('spark.sql.catalogImplementation', 'hive'),\n",
       " ('spark.rdd.compress', 'True'),\n",
       " ('spark.driver.host', '192.168.0.19'),\n",
       " ('spark.serializer.objectStreamReset', '100'),\n",
       " ('spark.app.id', 'local-1592509346247'),\n",
       " ('spark.executor.id', 'driver'),\n",
       " ('spark.submit.deployMode', 'client'),\n",
       " ('spark.ui.showConsoleProgress', 'true'),\n",
       " ('spark.app.name', 'Word Count'),\n",
       " ('spark.driver.port', '52912')]"
      ]
     },
     "execution_count": 6,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# Get all values as a list of key-value pairs\n",
    "spark.sparkContext.getConf().getAll()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "\n",
       "            <div>\n",
       "                <p><b>SparkSession - hive</b></p>\n",
       "                \n",
       "        <div>\n",
       "            <p><b>SparkContext</b></p>\n",
       "\n",
       "            <p><a href=\"http://192.168.0.19:4042\">Spark UI</a></p>\n",
       "\n",
       "            <dl>\n",
       "              <dt>Version</dt>\n",
       "                <dd><code>v2.4.5</code></dd>\n",
       "              <dt>Master</dt>\n",
       "                <dd><code>local[*]</code></dd>\n",
       "              <dt>AppName</dt>\n",
       "                <dd><code>PySparkShell</code></dd>\n",
       "            </dl>\n",
       "        </div>\n",
       "        \n",
       "            </div>\n",
       "        "
      ],
      "text/plain": [
       "<pyspark.sql.session.SparkSession at 0x7fd2b68cd510>"
      ]
     },
     "execution_count": 7,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "spark"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Read in the Data Set"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "DataFrame[Body: string, Id: bigint, Tags: string, Title: string, oneTag: string]"
      ]
     },
     "execution_count": 8,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "path = '/Users/yangweichle/Documents/Employment/TRAINING/DATA SCIENCE/Spark/Udacity_Spark for Big Data/Machine Learning with Spark/data/Train_onetag_small.json'\n",
    "\n",
    "# Loads JSON files and returns the results as a `DataFrame`\n",
    "# Note: path: string represents path to the JSON dataset, or a list of paths, or RDD of Strings storing JSON objects\n",
    "stack_overflow_data = spark.read.json(path=path)\n",
    "\n",
    "# Sets the storage level to persist the contents of the `DataFrame` across operations after the first time it is computed\n",
    "# This can only be used to assign a new storage level if the `DataFrame` does not have a storage level set yet\n",
    "# If no storage level is specified defaults to (C{MEMORY_AND_DISK})\n",
    "stack_overflow_data.persist()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "root\n",
      " |-- Body: string (nullable = true)\n",
      " |-- Id: long (nullable = true)\n",
      " |-- Tags: string (nullable = true)\n",
      " |-- Title: string (nullable = true)\n",
      " |-- oneTag: string (nullable = true)\n",
      "\n"
     ]
    }
   ],
   "source": [
    "# Prints out the schema in the tree format\n",
    "stack_overflow_data.printSchema()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "Row(Body=\"<p>I'd like to check if an uploaded file is an image file (e.g png, jpg, jpeg, gif, bmp) or another file. The problem is that I'm using Uploadify to upload the files, which changes the mime type and gives a 'text/octal' or something as the mime type, no matter which file type you upload.</p>\\n\\n<p>Is there a way to check if the uploaded file is an image apart from checking the file extension using PHP?</p>\\n\", Id=1, Tags='php image-processing file-upload upload mime-types', Title='How to check if an uploaded file is an image without mime type?', oneTag='php')"
      ]
     },
     "execution_count": 10,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# Returns the first ``n`` rows\n",
    "stack_overflow_data.head()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Tokenization\n",
    "\n",
    "Tokenization splits strings into separate words. Spark has a [Tokenizer](https://spark.apache.org/docs/latest/ml-features.html#tokenizer) class as well as RegexTokenizer, which allows for more control over the tokenization process."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "metadata": {},
   "outputs": [],
   "source": [
    "# A regex based tokenizer that extracts tokens either by using the provided regex pattern (in Java dialect) to \n",
    "#    split the text (default) or repeatedly matching the regex (if gaps is false)\n",
    "# Optional parameters also allow filtering tokens using a minimal length\n",
    "# It returns an array of strings that can be empty\n",
    "regexTokenizer = RegexTokenizer(inputCol='Body', outputCol='words', pattern='\\\\W')\n",
    "\n",
    "# Transforms the input dataset with optional parameters\n",
    "stack_overflow_data = regexTokenizer.transform(stack_overflow_data)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "Row(Body=\"<p>I'd like to check if an uploaded file is an image file (e.g png, jpg, jpeg, gif, bmp) or another file. The problem is that I'm using Uploadify to upload the files, which changes the mime type and gives a 'text/octal' or something as the mime type, no matter which file type you upload.</p>\\n\\n<p>Is there a way to check if the uploaded file is an image apart from checking the file extension using PHP?</p>\\n\", Id=1, Tags='php image-processing file-upload upload mime-types', Title='How to check if an uploaded file is an image without mime type?', oneTag='php', words=['p', 'i', 'd', 'like', 'to', 'check', 'if', 'an', 'uploaded', 'file', 'is', 'an', 'image', 'file', 'e', 'g', 'png', 'jpg', 'jpeg', 'gif', 'bmp', 'or', 'another', 'file', 'the', 'problem', 'is', 'that', 'i', 'm', 'using', 'uploadify', 'to', 'upload', 'the', 'files', 'which', 'changes', 'the', 'mime', 'type', 'and', 'gives', 'a', 'text', 'octal', 'or', 'something', 'as', 'the', 'mime', 'type', 'no', 'matter', 'which', 'file', 'type', 'you', 'upload', 'p', 'p', 'is', 'there', 'a', 'way', 'to', 'check', 'if', 'the', 'uploaded', 'file', 'is', 'an', 'image', 'apart', 'from', 'checking', 'the', 'file', 'extension', 'using', 'php', 'p'])"
      ]
     },
     "execution_count": 12,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# Returns the first ``n`` rows\n",
    "stack_overflow_data.head()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "#### Count the number of words in each body tag"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Create a user defined function (UDF)\n",
    "body_length = udf(lambda x: len(x), IntegerType())\n",
    "\n",
    "# Returns a new `DataFrame` by adding a column or replacing the existing column that has the same name\n",
    "stack_overflow_data = stack_overflow_data.withColumn('BodyLength', body_length(stack_overflow_data.words))"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "#### Count the number of paragraphs and links in each body tag"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 14,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Create a user defined function (UDF)\n",
    "number_of_paragraphs = udf(lambda x: len(re.findall('</p>', x)), IntegerType())\n",
    "number_of_links = udf(lambda x: len(re.findall('</a>', x)), IntegerType())\n",
    "\n",
    "# Returns a new `DataFrame` by adding a column or replacing the existing column that has the same name\n",
    "stack_overflow_data = stack_overflow_data.withColumn('NumParagraphs', number_of_paragraphs(stack_overflow_data.Body))\n",
    "stack_overflow_data = stack_overflow_data.withColumn('NumLinks', number_of_links(stack_overflow_data.Body))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 15,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "[Row(Body=\"<p>I'd like to check if an uploaded file is an image file (e.g png, jpg, jpeg, gif, bmp) or another file. The problem is that I'm using Uploadify to upload the files, which changes the mime type and gives a 'text/octal' or something as the mime type, no matter which file type you upload.</p>\\n\\n<p>Is there a way to check if the uploaded file is an image apart from checking the file extension using PHP?</p>\\n\", Id=1, Tags='php image-processing file-upload upload mime-types', Title='How to check if an uploaded file is an image without mime type?', oneTag='php', words=['p', 'i', 'd', 'like', 'to', 'check', 'if', 'an', 'uploaded', 'file', 'is', 'an', 'image', 'file', 'e', 'g', 'png', 'jpg', 'jpeg', 'gif', 'bmp', 'or', 'another', 'file', 'the', 'problem', 'is', 'that', 'i', 'm', 'using', 'uploadify', 'to', 'upload', 'the', 'files', 'which', 'changes', 'the', 'mime', 'type', 'and', 'gives', 'a', 'text', 'octal', 'or', 'something', 'as', 'the', 'mime', 'type', 'no', 'matter', 'which', 'file', 'type', 'you', 'upload', 'p', 'p', 'is', 'there', 'a', 'way', 'to', 'check', 'if', 'the', 'uploaded', 'file', 'is', 'an', 'image', 'apart', 'from', 'checking', 'the', 'file', 'extension', 'using', 'php', 'p'], BodyLength=83, NumParagraphs=2, NumLinks=0),\n",
       " Row(Body='<p>In my favorite editor (vim), I regularly use ctrl-w to execute a certain action. Now, it quite often happens to me that firefox is the active window (on windows) while I still look at vim (thinking vim is the active window) and press ctrl-w which closes firefox. This is not what I want. Is there a way to stop ctrl-w from closing firefox?</p>\\n\\n<p>Rene</p>\\n', Id=2, Tags='firefox', Title='How can I prevent firefox from closing when I press ctrl-w', oneTag='firefox', words=['p', 'in', 'my', 'favorite', 'editor', 'vim', 'i', 'regularly', 'use', 'ctrl', 'w', 'to', 'execute', 'a', 'certain', 'action', 'now', 'it', 'quite', 'often', 'happens', 'to', 'me', 'that', 'firefox', 'is', 'the', 'active', 'window', 'on', 'windows', 'while', 'i', 'still', 'look', 'at', 'vim', 'thinking', 'vim', 'is', 'the', 'active', 'window', 'and', 'press', 'ctrl', 'w', 'which', 'closes', 'firefox', 'this', 'is', 'not', 'what', 'i', 'want', 'is', 'there', 'a', 'way', 'to', 'stop', 'ctrl', 'w', 'from', 'closing', 'firefox', 'p', 'p', 'rene', 'p'], BodyLength=71, NumParagraphs=2, NumLinks=0)]"
      ]
     },
     "execution_count": 15,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# Returns the first ``n`` rows\n",
    "stack_overflow_data.head(2)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### VectorAssembler\n",
    "\n",
    "Combine the body length, number of paragraphs, and number of links columns into a vector"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 16,
   "metadata": {},
   "outputs": [],
   "source": [
    "# A feature transformer that merges multiple columns into a vector column\n",
    "vecAssembler = VectorAssembler(inputCols=['BodyLength', 'NumParagraphs', 'NumLinks'], outputCol='NumFeatures')\n",
    "\n",
    "# Transforms the input dataset with optional parameters\n",
    "stack_overflow_data = vecAssembler.transform(stack_overflow_data)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 17,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "Row(Body=\"<p>I'd like to check if an uploaded file is an image file (e.g png, jpg, jpeg, gif, bmp) or another file. The problem is that I'm using Uploadify to upload the files, which changes the mime type and gives a 'text/octal' or something as the mime type, no matter which file type you upload.</p>\\n\\n<p>Is there a way to check if the uploaded file is an image apart from checking the file extension using PHP?</p>\\n\", Id=1, Tags='php image-processing file-upload upload mime-types', Title='How to check if an uploaded file is an image without mime type?', oneTag='php', words=['p', 'i', 'd', 'like', 'to', 'check', 'if', 'an', 'uploaded', 'file', 'is', 'an', 'image', 'file', 'e', 'g', 'png', 'jpg', 'jpeg', 'gif', 'bmp', 'or', 'another', 'file', 'the', 'problem', 'is', 'that', 'i', 'm', 'using', 'uploadify', 'to', 'upload', 'the', 'files', 'which', 'changes', 'the', 'mime', 'type', 'and', 'gives', 'a', 'text', 'octal', 'or', 'something', 'as', 'the', 'mime', 'type', 'no', 'matter', 'which', 'file', 'type', 'you', 'upload', 'p', 'p', 'is', 'there', 'a', 'way', 'to', 'check', 'if', 'the', 'uploaded', 'file', 'is', 'an', 'image', 'apart', 'from', 'checking', 'the', 'file', 'extension', 'using', 'php', 'p'], BodyLength=83, NumParagraphs=2, NumLinks=0, NumFeatures=DenseVector([83.0, 2.0, 0.0]))"
      ]
     },
     "execution_count": 17,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# Returns the first ``n`` rows\n",
    "stack_overflow_data.head()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Normalize the Vectors"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 18,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Normalize a vector to have unit norm using the given p-norm\n",
    "normalizer = Normalizer(inputCol='NumFeatures', outputCol='ScaledNumFeatures')\n",
    "\n",
    "# Transforms the input dataset with optional parameters\n",
    "stack_overflow_data = normalizer.transform(stack_overflow_data)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 19,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "[Row(Body=\"<p>I'd like to check if an uploaded file is an image file (e.g png, jpg, jpeg, gif, bmp) or another file. The problem is that I'm using Uploadify to upload the files, which changes the mime type and gives a 'text/octal' or something as the mime type, no matter which file type you upload.</p>\\n\\n<p>Is there a way to check if the uploaded file is an image apart from checking the file extension using PHP?</p>\\n\", Id=1, Tags='php image-processing file-upload upload mime-types', Title='How to check if an uploaded file is an image without mime type?', oneTag='php', words=['p', 'i', 'd', 'like', 'to', 'check', 'if', 'an', 'uploaded', 'file', 'is', 'an', 'image', 'file', 'e', 'g', 'png', 'jpg', 'jpeg', 'gif', 'bmp', 'or', 'another', 'file', 'the', 'problem', 'is', 'that', 'i', 'm', 'using', 'uploadify', 'to', 'upload', 'the', 'files', 'which', 'changes', 'the', 'mime', 'type', 'and', 'gives', 'a', 'text', 'octal', 'or', 'something', 'as', 'the', 'mime', 'type', 'no', 'matter', 'which', 'file', 'type', 'you', 'upload', 'p', 'p', 'is', 'there', 'a', 'way', 'to', 'check', 'if', 'the', 'uploaded', 'file', 'is', 'an', 'image', 'apart', 'from', 'checking', 'the', 'file', 'extension', 'using', 'php', 'p'], BodyLength=83, NumParagraphs=2, NumLinks=0, NumFeatures=DenseVector([83.0, 2.0, 0.0]), ScaledNumFeatures=DenseVector([0.9997, 0.0241, 0.0])),\n",
       " Row(Body='<p>In my favorite editor (vim), I regularly use ctrl-w to execute a certain action. Now, it quite often happens to me that firefox is the active window (on windows) while I still look at vim (thinking vim is the active window) and press ctrl-w which closes firefox. This is not what I want. Is there a way to stop ctrl-w from closing firefox?</p>\\n\\n<p>Rene</p>\\n', Id=2, Tags='firefox', Title='How can I prevent firefox from closing when I press ctrl-w', oneTag='firefox', words=['p', 'in', 'my', 'favorite', 'editor', 'vim', 'i', 'regularly', 'use', 'ctrl', 'w', 'to', 'execute', 'a', 'certain', 'action', 'now', 'it', 'quite', 'often', 'happens', 'to', 'me', 'that', 'firefox', 'is', 'the', 'active', 'window', 'on', 'windows', 'while', 'i', 'still', 'look', 'at', 'vim', 'thinking', 'vim', 'is', 'the', 'active', 'window', 'and', 'press', 'ctrl', 'w', 'which', 'closes', 'firefox', 'this', 'is', 'not', 'what', 'i', 'want', 'is', 'there', 'a', 'way', 'to', 'stop', 'ctrl', 'w', 'from', 'closing', 'firefox', 'p', 'p', 'rene', 'p'], BodyLength=71, NumParagraphs=2, NumLinks=0, NumFeatures=DenseVector([71.0, 2.0, 0.0]), ScaledNumFeatures=DenseVector([0.9996, 0.0282, 0.0]))]"
      ]
     },
     "execution_count": 19,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# Returns the first ``n`` rows\n",
    "stack_overflow_data.head(2)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Scale the Vectors"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 20,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Standardizes features by removing the mean and scaling to unit variance using column  summary statistics \n",
    "#    on the samples in the training set\n",
    "# The \"unit std\" is computed using the `corrected sample standard deviation`, which is computed as the \n",
    "#    square root of the unbiased sample variance\n",
    "standardScaler = StandardScaler(inputCol='NumFeatures', outputCol='ScaledNumFeatures2', withStd=True)\n",
    "\n",
    "# Fits a model to the input dataset with optional parameters\n",
    "scalerModel = standardScaler.fit(stack_overflow_data)\n",
    "\n",
    "# Transforms the input dataset with optional parameters\n",
    "stack_overflow_data = scalerModel.transform(stack_overflow_data)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 21,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "[Row(Body=\"<p>I'd like to check if an uploaded file is an image file (e.g png, jpg, jpeg, gif, bmp) or another file. The problem is that I'm using Uploadify to upload the files, which changes the mime type and gives a 'text/octal' or something as the mime type, no matter which file type you upload.</p>\\n\\n<p>Is there a way to check if the uploaded file is an image apart from checking the file extension using PHP?</p>\\n\", Id=1, Tags='php image-processing file-upload upload mime-types', Title='How to check if an uploaded file is an image without mime type?', oneTag='php', words=['p', 'i', 'd', 'like', 'to', 'check', 'if', 'an', 'uploaded', 'file', 'is', 'an', 'image', 'file', 'e', 'g', 'png', 'jpg', 'jpeg', 'gif', 'bmp', 'or', 'another', 'file', 'the', 'problem', 'is', 'that', 'i', 'm', 'using', 'uploadify', 'to', 'upload', 'the', 'files', 'which', 'changes', 'the', 'mime', 'type', 'and', 'gives', 'a', 'text', 'octal', 'or', 'something', 'as', 'the', 'mime', 'type', 'no', 'matter', 'which', 'file', 'type', 'you', 'upload', 'p', 'p', 'is', 'there', 'a', 'way', 'to', 'check', 'if', 'the', 'uploaded', 'file', 'is', 'an', 'image', 'apart', 'from', 'checking', 'the', 'file', 'extension', 'using', 'php', 'p'], BodyLength=83, NumParagraphs=2, NumLinks=0, NumFeatures=DenseVector([83.0, 2.0, 0.0]), ScaledNumFeatures=DenseVector([0.9997, 0.0241, 0.0]), ScaledNumFeatures2=DenseVector([0.4325, 0.7037, 0.0])),\n",
       " Row(Body='<p>In my favorite editor (vim), I regularly use ctrl-w to execute a certain action. Now, it quite often happens to me that firefox is the active window (on windows) while I still look at vim (thinking vim is the active window) and press ctrl-w which closes firefox. This is not what I want. Is there a way to stop ctrl-w from closing firefox?</p>\\n\\n<p>Rene</p>\\n', Id=2, Tags='firefox', Title='How can I prevent firefox from closing when I press ctrl-w', oneTag='firefox', words=['p', 'in', 'my', 'favorite', 'editor', 'vim', 'i', 'regularly', 'use', 'ctrl', 'w', 'to', 'execute', 'a', 'certain', 'action', 'now', 'it', 'quite', 'often', 'happens', 'to', 'me', 'that', 'firefox', 'is', 'the', 'active', 'window', 'on', 'windows', 'while', 'i', 'still', 'look', 'at', 'vim', 'thinking', 'vim', 'is', 'the', 'active', 'window', 'and', 'press', 'ctrl', 'w', 'which', 'closes', 'firefox', 'this', 'is', 'not', 'what', 'i', 'want', 'is', 'there', 'a', 'way', 'to', 'stop', 'ctrl', 'w', 'from', 'closing', 'firefox', 'p', 'p', 'rene', 'p'], BodyLength=71, NumParagraphs=2, NumLinks=0, NumFeatures=DenseVector([71.0, 2.0, 0.0]), ScaledNumFeatures=DenseVector([0.9996, 0.0282, 0.0]), ScaledNumFeatures2=DenseVector([0.3699, 0.7037, 0.0]))]"
      ]
     },
     "execution_count": 21,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# Returns the first ``n`` rows\n",
    "stack_overflow_data.head(2)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Text Processing\n",
    "\n",
    "Find the term frequencies of the words."
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### CountVectorizer"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 22,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Extracts a vocabulary from document collections and generates a `CountVectorizerModel`\n",
    "cv = CountVectorizer(inputCol='words', outputCol='TF', vocabSize=1000)\n",
    "\n",
    "# Fits a model to the input dataset with optional parameters\n",
    "cvModel = cv.fit(stack_overflow_data)\n",
    "\n",
    "# Transforms the input dataset with optional parameters\n",
    "stack_overflow_data = cvModel.transform(stack_overflow_data)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 23,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "[Row(Body=\"<p>I'd like to check if an uploaded file is an image file (e.g png, jpg, jpeg, gif, bmp) or another file. The problem is that I'm using Uploadify to upload the files, which changes the mime type and gives a 'text/octal' or something as the mime type, no matter which file type you upload.</p>\\n\\n<p>Is there a way to check if the uploaded file is an image apart from checking the file extension using PHP?</p>\\n\", Id=1, Tags='php image-processing file-upload upload mime-types', Title='How to check if an uploaded file is an image without mime type?', oneTag='php', words=['p', 'i', 'd', 'like', 'to', 'check', 'if', 'an', 'uploaded', 'file', 'is', 'an', 'image', 'file', 'e', 'g', 'png', 'jpg', 'jpeg', 'gif', 'bmp', 'or', 'another', 'file', 'the', 'problem', 'is', 'that', 'i', 'm', 'using', 'uploadify', 'to', 'upload', 'the', 'files', 'which', 'changes', 'the', 'mime', 'type', 'and', 'gives', 'a', 'text', 'octal', 'or', 'something', 'as', 'the', 'mime', 'type', 'no', 'matter', 'which', 'file', 'type', 'you', 'upload', 'p', 'p', 'is', 'there', 'a', 'way', 'to', 'check', 'if', 'the', 'uploaded', 'file', 'is', 'an', 'image', 'apart', 'from', 'checking', 'the', 'file', 'extension', 'using', 'php', 'p'], BodyLength=83, NumParagraphs=2, NumLinks=0, NumFeatures=DenseVector([83.0, 2.0, 0.0]), ScaledNumFeatures=DenseVector([0.9997, 0.0241, 0.0]), ScaledNumFeatures2=DenseVector([0.4325, 0.7037, 0.0]), TF=SparseVector(1000, {0: 4.0, 1: 6.0, 2: 2.0, 3: 3.0, 5: 2.0, 8: 4.0, 9: 1.0, 15: 1.0, 21: 2.0, 28: 1.0, 31: 1.0, 35: 3.0, 36: 1.0, 43: 2.0, 45: 2.0, 48: 1.0, 51: 1.0, 57: 6.0, 61: 2.0, 71: 1.0, 78: 1.0, 84: 3.0, 86: 1.0, 94: 1.0, 97: 1.0, 99: 1.0, 100: 1.0, 115: 1.0, 147: 2.0, 152: 1.0, 169: 1.0, 241: 1.0, 283: 1.0, 306: 1.0, 350: 2.0, 490: 1.0, 578: 1.0, 759: 1.0, 832: 2.0}))]"
      ]
     },
     "execution_count": 23,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# Returns the first ``num`` rows as a `list` of `Row`\n",
    "stack_overflow_data.take(1)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 24,
   "metadata": {
    "scrolled": true
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "['p',\n",
       " 'the',\n",
       " 'i',\n",
       " 'to',\n",
       " 'code',\n",
       " 'a',\n",
       " 'gt',\n",
       " 'lt',\n",
       " 'is',\n",
       " 'and',\n",
       " 'pre',\n",
       " 'in',\n",
       " 'this',\n",
       " 'of',\n",
       " 'it',\n",
       " 'that',\n",
       " 'for',\n",
       " '0',\n",
       " '1',\n",
       " 'have',\n",
       " 'my',\n",
       " 'if',\n",
       " 'on',\n",
       " 'but',\n",
       " 'with',\n",
       " 'can',\n",
       " 'not',\n",
       " 'be',\n",
       " 'as',\n",
       " 't',\n",
       " 'li',\n",
       " 'from',\n",
       " '2',\n",
       " 's',\n",
       " 'http',\n",
       " 'an',\n",
       " 'm',\n",
       " 'strong',\n",
       " 'new',\n",
       " 'how',\n",
       " 'do',\n",
       " 'com',\n",
       " 'so',\n",
       " 'or',\n",
       " 'at',\n",
       " 'using',\n",
       " 'when',\n",
       " 'am',\n",
       " 'like',\n",
       " 'class',\n",
       " 'id',\n",
       " 'there',\n",
       " 'get',\n",
       " 'are',\n",
       " 'name',\n",
       " 'what',\n",
       " 'any',\n",
       " 'file',\n",
       " 'string',\n",
       " 'data',\n",
       " 'all',\n",
       " 'which',\n",
       " 'want',\n",
       " 'would',\n",
       " 'amp',\n",
       " 'use',\n",
       " 'java',\n",
       " 'function',\n",
       " 'public',\n",
       " 'some',\n",
       " '3',\n",
       " 'text',\n",
       " 'error',\n",
       " 'android',\n",
       " 'value',\n",
       " 'c',\n",
       " 'x',\n",
       " 'href',\n",
       " 'you',\n",
       " 'one',\n",
       " 'by',\n",
       " 'user',\n",
       " 'me',\n",
       " 'server',\n",
       " 'type',\n",
       " 'here',\n",
       " 'way',\n",
       " 'return',\n",
       " 'int',\n",
       " 'will',\n",
       " 'div',\n",
       " 'need',\n",
       " 'then',\n",
       " 'set',\n",
       " 'e',\n",
       " 'system',\n",
       " 'has',\n",
       " 'problem',\n",
       " 'out',\n",
       " 'php',\n",
       " 'no',\n",
       " 'just',\n",
       " '4',\n",
       " 'org',\n",
       " 'know',\n",
       " 'html',\n",
       " 'only',\n",
       " 'where',\n",
       " 'page',\n",
       " 'application',\n",
       " '5',\n",
       " 'thanks',\n",
       " 'var',\n",
       " 'br',\n",
       " 'we',\n",
       " 'd',\n",
       " 'should',\n",
       " 'does',\n",
       " 'add',\n",
       " 'n',\n",
       " 'true',\n",
       " 've',\n",
       " 'void',\n",
       " 'em',\n",
       " 'was',\n",
       " 'rel',\n",
       " 'work',\n",
       " 'time',\n",
       " 'other',\n",
       " '10',\n",
       " 'app',\n",
       " 'null',\n",
       " 'method',\n",
       " 'b',\n",
       " 'table',\n",
       " 'list',\n",
       " 'now',\n",
       " 'into',\n",
       " 'help',\n",
       " 'end',\n",
       " 'trying',\n",
       " 'following',\n",
       " 'object',\n",
       " 'view',\n",
       " 'nofollow',\n",
       " 'up',\n",
       " 'example',\n",
       " 'image',\n",
       " 'same',\n",
       " 'create',\n",
       " 'also',\n",
       " 'each',\n",
       " 'something',\n",
       " 'www',\n",
       " 'web',\n",
       " 'first',\n",
       " 'array',\n",
       " 'line',\n",
       " 'script',\n",
       " 'find',\n",
       " 'don',\n",
       " 'run',\n",
       " 'could',\n",
       " 'select',\n",
       " 'about',\n",
       " 'test',\n",
       " 'make',\n",
       " 'form',\n",
       " 'r',\n",
       " 'files',\n",
       " 'tried',\n",
       " 'ul',\n",
       " 'net',\n",
       " 'url',\n",
       " 'td',\n",
       " 'self',\n",
       " 'input',\n",
       " 'windows',\n",
       " 'button',\n",
       " 'see',\n",
       " 'blockquote',\n",
       " 'database',\n",
       " 'question',\n",
       " 'content',\n",
       " 'else',\n",
       " 'more',\n",
       " 'works',\n",
       " 'xml',\n",
       " '6',\n",
       " '00',\n",
       " 'two',\n",
       " '8',\n",
       " 'after',\n",
       " 'they',\n",
       " 'possible',\n",
       " 'false',\n",
       " 'right',\n",
       " 'them',\n",
       " 'y',\n",
       " 'working',\n",
       " '7',\n",
       " 'width',\n",
       " 'main',\n",
       " 'src',\n",
       " 'try',\n",
       " 'private',\n",
       " 'however',\n",
       " 'version',\n",
       " 'number',\n",
       " 'f',\n",
       " 'result',\n",
       " 'these',\n",
       " 'because',\n",
       " 'project',\n",
       " 'message',\n",
       " 'key',\n",
       " 'why',\n",
       " 'doesn',\n",
       " 'used',\n",
       " 'please',\n",
       " 'query',\n",
       " 'import',\n",
       " 'size',\n",
       " 'item',\n",
       " 'call',\n",
       " 'show',\n",
       " 'while',\n",
       " 'title',\n",
       " 'found',\n",
       " 'been',\n",
       " 'anyone',\n",
       " 'change',\n",
       " 'post',\n",
       " 'document',\n",
       " 'users',\n",
       " 'different',\n",
       " 'its',\n",
       " 'start',\n",
       " 'able',\n",
       " 'log',\n",
       " 'access',\n",
       " 'another',\n",
       " 'event',\n",
       " 'case',\n",
       " 'request',\n",
       " 'values',\n",
       " 'update',\n",
       " 'client',\n",
       " 'edit',\n",
       " 'index',\n",
       " '9',\n",
       " 'service',\n",
       " 'read',\n",
       " 'without',\n",
       " 'source',\n",
       " 'javascript',\n",
       " 'left',\n",
       " 'style',\n",
       " 'open',\n",
       " 'jquery',\n",
       " 'img',\n",
       " 'running',\n",
       " 'row',\n",
       " 'h',\n",
       " 'display',\n",
       " 'fine',\n",
       " 'write',\n",
       " 'site',\n",
       " 'google',\n",
       " 'seems',\n",
       " 'height',\n",
       " 'click',\n",
       " 'date',\n",
       " '12',\n",
       " 'static',\n",
       " 'etc',\n",
       " 'option',\n",
       " 'path',\n",
       " 'output',\n",
       " 'property',\n",
       " '20',\n",
       " 'doing',\n",
       " 'model',\n",
       " 'g',\n",
       " 'default',\n",
       " 'link',\n",
       " 'than',\n",
       " 'through',\n",
       " 'echo',\n",
       " 'below',\n",
       " 'include',\n",
       " 'even',\n",
       " 'solution',\n",
       " 'lib',\n",
       " '11',\n",
       " 'sql',\n",
       " 'questions',\n",
       " 'still',\n",
       " 'program',\n",
       " 'such',\n",
       " 'library',\n",
       " 'getting',\n",
       " 'exception',\n",
       " 'created',\n",
       " 'simple',\n",
       " 'context',\n",
       " 'png',\n",
       " 'your',\n",
       " 'very',\n",
       " 'before',\n",
       " 'apache',\n",
       " 'both',\n",
       " 'ol',\n",
       " 'sure',\n",
       " '100',\n",
       " 'order',\n",
       " 'asp',\n",
       " 'command',\n",
       " 'field',\n",
       " 'color',\n",
       " 'window',\n",
       " 'images',\n",
       " 'column',\n",
       " 'load',\n",
       " 'having',\n",
       " 'thread',\n",
       " 'background',\n",
       " 'think',\n",
       " 'js',\n",
       " 'wrong',\n",
       " 'go',\n",
       " 'point',\n",
       " 'element',\n",
       " 'process',\n",
       " 'length',\n",
       " 'really',\n",
       " 'tr',\n",
       " 'span',\n",
       " 'being',\n",
       " 'every',\n",
       " 'back',\n",
       " 'current',\n",
       " 'called',\n",
       " 'css',\n",
       " 'label',\n",
       " 'action',\n",
       " 'issue',\n",
       " 'many',\n",
       " 'info',\n",
       " 'stack',\n",
       " 'check',\n",
       " 'got',\n",
       " 'top',\n",
       " 'since',\n",
       " 'connection',\n",
       " 'looking',\n",
       " 'put',\n",
       " 'second',\n",
       " 'search',\n",
       " 'db',\n",
       " 'local',\n",
       " 'over',\n",
       " 'email',\n",
       " 'above',\n",
       " 'password',\n",
       " 'done',\n",
       " 'api',\n",
       " 'between',\n",
       " 'response',\n",
       " 'build',\n",
       " 'cannot',\n",
       " 'alt',\n",
       " 'print',\n",
       " 'j',\n",
       " 'well',\n",
       " 'body',\n",
       " 'directory',\n",
       " 'count',\n",
       " 'description',\n",
       " 'location',\n",
       " 'information',\n",
       " 'next',\n",
       " 'address',\n",
       " 'root',\n",
       " 'good',\n",
       " '01',\n",
       " 'let',\n",
       " 'control',\n",
       " 'o',\n",
       " 'microsoft',\n",
       " 'part',\n",
       " 'map',\n",
       " 'advance',\n",
       " 'instead',\n",
       " 'our',\n",
       " 'v',\n",
       " 'much',\n",
       " 'their',\n",
       " 'best',\n",
       " 'position',\n",
       " '2012',\n",
       " 'idea',\n",
       " 'custom',\n",
       " 'format',\n",
       " 'mysql',\n",
       " 'already',\n",
       " 'say',\n",
       " 'long',\n",
       " 'instance',\n",
       " 'variable',\n",
       " 'send',\n",
       " '13',\n",
       " '16',\n",
       " 'may',\n",
       " 'currently',\n",
       " 'results',\n",
       " 'inside',\n",
       " '15',\n",
       " 'k',\n",
       " 'header',\n",
       " 'enter',\n",
       " 'items',\n",
       " 'correct',\n",
       " 'home',\n",
       " 'controller',\n",
       " 'domain',\n",
       " 'node',\n",
       " 'z',\n",
       " 'override',\n",
       " 'based',\n",
       " '30',\n",
       " 'seem',\n",
       " 'group',\n",
       " 'options',\n",
       " 'last',\n",
       " 'ui',\n",
       " 'added',\n",
       " 'screen',\n",
       " 'someone',\n",
       " 'folder',\n",
       " 'save',\n",
       " 'session',\n",
       " 'website',\n",
       " 'close',\n",
       " 'stackoverflow',\n",
       " 'better',\n",
       " 'username',\n",
       " 'python',\n",
       " 'bit',\n",
       " 'menu',\n",
       " 'json',\n",
       " 'box',\n",
       " 'login',\n",
       " 'console',\n",
       " 'via',\n",
       " 'l',\n",
       " 'usr',\n",
       " 'looks',\n",
       " 'anything',\n",
       " 'def',\n",
       " 'none',\n",
       " 'https',\n",
       " 'char',\n",
       " 'lang',\n",
       " 'within',\n",
       " 'multiple',\n",
       " 'activity',\n",
       " 'everything',\n",
       " 'appreciated',\n",
       " 'tag',\n",
       " 'insert',\n",
       " 'ajax',\n",
       " 'had',\n",
       " 'config',\n",
       " 'again',\n",
       " 'understand',\n",
       " 'parent',\n",
       " 'final',\n",
       " 'install',\n",
       " 'catch',\n",
       " 'browser',\n",
       " 'objects',\n",
       " 'store',\n",
       " 'os',\n",
       " 'ideas',\n",
       " 'double',\n",
       " 'contains',\n",
       " 'thing',\n",
       " 'look',\n",
       " 'given',\n",
       " 'jpg',\n",
       " 'template',\n",
       " 'font',\n",
       " 'reference',\n",
       " 'those',\n",
       " 'answer',\n",
       " 'going',\n",
       " 'connect',\n",
       " 'debug',\n",
       " 'imgur',\n",
       " 'frame',\n",
       " 'rows',\n",
       " 'down',\n",
       " 'ruby',\n",
       " 'foo',\n",
       " 'around',\n",
       " 'always',\n",
       " 'too',\n",
       " 'state',\n",
       " 'block',\n",
       " 'remove',\n",
       " 'status',\n",
       " 'intent',\n",
       " 'yes',\n",
       " 'must',\n",
       " 'errors',\n",
       " 'thank',\n",
       " '08',\n",
       " 'layout',\n",
       " 'en',\n",
       " 'did',\n",
       " 'far',\n",
       " 'u',\n",
       " 'core',\n",
       " 'memory',\n",
       " 'most',\n",
       " 'installed',\n",
       " 'single',\n",
       " 'w',\n",
       " 'framework',\n",
       " 'bar',\n",
       " 'linux',\n",
       " 'machine',\n",
       " 'specific',\n",
       " 'loop',\n",
       " 'give',\n",
       " '14',\n",
       " 'delete',\n",
       " 'begin',\n",
       " 'take',\n",
       " 'ip',\n",
       " 'lot',\n",
       " 'returns',\n",
       " '02',\n",
       " 'facebook',\n",
       " 're',\n",
       " 'device',\n",
       " 'things',\n",
       " 'creating',\n",
       " 'base',\n",
       " 'ok',\n",
       " 'nothing',\n",
       " 'missing',\n",
       " 'module',\n",
       " 'fields',\n",
       " 'host',\n",
       " 'own',\n",
       " 'uses',\n",
       " 'support',\n",
       " 'androidruntime',\n",
       " 'might',\n",
       " 'similar',\n",
       " 'float',\n",
       " 'integer',\n",
       " 'actually',\n",
       " 'figure',\n",
       " '50',\n",
       " 'configuration',\n",
       " '2010',\n",
       " 'alert',\n",
       " 'tell',\n",
       " 'us',\n",
       " 'execute',\n",
       " 'std',\n",
       " 'tostring',\n",
       " 'few',\n",
       " 'empty',\n",
       " 'note',\n",
       " 'changes',\n",
       " 'xmlns',\n",
       " 'pass',\n",
       " 'failed',\n",
       " 'off',\n",
       " 'settings',\n",
       " 'network',\n",
       " 'either',\n",
       " 'package',\n",
       " 'interface',\n",
       " '2011',\n",
       " 'eclipse',\n",
       " 'mode',\n",
       " 'plugin',\n",
       " '25',\n",
       " 'elements',\n",
       " 'methods',\n",
       " 'classes',\n",
       " 'rb',\n",
       " 'head',\n",
       " 'shows',\n",
       " 'properties',\n",
       " 'video',\n",
       " '18',\n",
       " 'const',\n",
       " 'port',\n",
       " 'pages',\n",
       " 'break',\n",
       " 'columns',\n",
       " 'made',\n",
       " 'bin',\n",
       " 'cell',\n",
       " '23',\n",
       " 'functions',\n",
       " 'allow',\n",
       " '21',\n",
       " 'nbsp',\n",
       " 'generated',\n",
       " 'q',\n",
       " 'submit',\n",
       " 'once',\n",
       " 'setup',\n",
       " 'product',\n",
       " 'grid',\n",
       " '22',\n",
       " 'reason',\n",
       " 'copy',\n",
       " 'h2',\n",
       " 'times',\n",
       " 'aspx',\n",
       " 'sub',\n",
       " 'auto',\n",
       " 'target',\n",
       " 'dev',\n",
       " 'available',\n",
       " '04',\n",
       " 'jar',\n",
       " 'keep',\n",
       " 'filter',\n",
       " 'required',\n",
       " 'side',\n",
       " 'convert',\n",
       " 'tables',\n",
       " 'handle',\n",
       " 'center',\n",
       " 'println',\n",
       " 'task',\n",
       " 'selected',\n",
       " 'join',\n",
       " 'setting',\n",
       " 'security',\n",
       " 'gets',\n",
       " 'category',\n",
       " '17',\n",
       " '05',\n",
       " 'lines',\n",
       " 'under',\n",
       " 'though',\n",
       " 'boolean',\n",
       " 'gems',\n",
       " 'entity',\n",
       " 'localhost',\n",
       " 'isn',\n",
       " 'space',\n",
       " 'admin',\n",
       " 'sun',\n",
       " 'fix',\n",
       " 'dll',\n",
       " 'implement',\n",
       " 'internal',\n",
       " 'wondering',\n",
       " 'nil',\n",
       " 'init',\n",
       " 'val',\n",
       " '09',\n",
       " 'parameters',\n",
       " '19',\n",
       " 'binding',\n",
       " 'rails',\n",
       " 'txt',\n",
       " 'maybe',\n",
       " 'level',\n",
       " 'super',\n",
       " 'several',\n",
       " 'child',\n",
       " 'defined',\n",
       " 'non',\n",
       " '03',\n",
       " 'parameter',\n",
       " 'place',\n",
       " '24',\n",
       " 'sort',\n",
       " 'needs',\n",
       " 'language',\n",
       " 'adding',\n",
       " 'py',\n",
       " 'dialog',\n",
       " 'cache',\n",
       " 'statement',\n",
       " 'section',\n",
       " 'kind',\n",
       " 'problems',\n",
       " 'total',\n",
       " '27',\n",
       " 'margin',\n",
       " 'utf',\n",
       " 'filename',\n",
       " 'nsstring',\n",
       " 'writing',\n",
       " 'border',\n",
       " 'account',\n",
       " 'computer',\n",
       " 'changed',\n",
       " 'correctly',\n",
       " 'match',\n",
       " '255',\n",
       " 'tab',\n",
       " 'bundle',\n",
       " 'define',\n",
       " 'didn',\n",
       " 'warning',\n",
       " 'basically',\n",
       " 'solve',\n",
       " 'release',\n",
       " 'protected',\n",
       " 'hr',\n",
       " 'structure',\n",
       " 'replace',\n",
       " 'frac',\n",
       " 'software',\n",
       " 'foreach',\n",
       " 'extends',\n",
       " '29',\n",
       " 'thought',\n",
       " 'full',\n",
       " 'byte',\n",
       " 'word',\n",
       " 'stored',\n",
       " 'ie',\n",
       " '_',\n",
       " 'day',\n",
       " 'sender',\n",
       " 'record',\n",
       " 'clear',\n",
       " 'never',\n",
       " 'services',\n",
       " 'were',\n",
       " 'pdf',\n",
       " '32',\n",
       " 'stop',\n",
       " '07',\n",
       " 'resources',\n",
       " 'models',\n",
       " 'springframework',\n",
       " 'move',\n",
       " 'alloc',\n",
       " 'people',\n",
       " 'started',\n",
       " 'checked',\n",
       " 'virtual',\n",
       " 'attribute',\n",
       " 'gives',\n",
       " 'suggestions',\n",
       " 'container',\n",
       " 'ubuntu',\n",
       " 'numbers',\n",
       " 'old',\n",
       " 'mail',\n",
       " 'sample',\n",
       " 'unknown',\n",
       " 'bool',\n",
       " 'small',\n",
       " 'align',\n",
       " 'collection',\n",
       " 'success',\n",
       " 'visual',\n",
       " 'person',\n",
       " 'compile',\n",
       " 'mean',\n",
       " 'standard',\n",
       " 'encoding',\n",
       " 'param',\n",
       " 'iphone',\n",
       " 'append',\n",
       " 'projects',\n",
       " 'events',\n",
       " 'bytes',\n",
       " 'str',\n",
       " '06',\n",
       " 'rather',\n",
       " 'written',\n",
       " 'reading',\n",
       " 'resource',\n",
       " 'wrap_content',\n",
       " 'exists',\n",
       " 'details',\n",
       " 'certain',\n",
       " 'widget',\n",
       " 'handler',\n",
       " 'download',\n",
       " 'parse',\n",
       " 'stuff',\n",
       " '40',\n",
       " 'until',\n",
       " 'documentation',\n",
       " 'expected',\n",
       " 'variables',\n",
       " 'socket',\n",
       " 'git',\n",
       " 'little',\n",
       " 'servlet',\n",
       " 'hello',\n",
       " 'th',\n",
       " 'modules',\n",
       " 'environment',\n",
       " 'generate',\n",
       " 'says',\n",
       " 'development',\n",
       " 'easy',\n",
       " 'simply',\n",
       " 'large',\n",
       " 'three',\n",
       " '31',\n",
       " 'exe',\n",
       " 'game',\n",
       " 'approach',\n",
       " 'calls',\n",
       " 'args',\n",
       " 'normal',\n",
       " 'bottom',\n",
       " 'global',\n",
       " 'views',\n",
       " 'types',\n",
       " 'javax',\n",
       " 'upload',\n",
       " 'invoke',\n",
       " 'tags',\n",
       " 'hibernate',\n",
       " 'switch',\n",
       " 'stream',\n",
       " 'onclick',\n",
       " 'making',\n",
       " 'remote',\n",
       " 'layout_width',\n",
       " 'layout_height',\n",
       " 'entry',\n",
       " 'im',\n",
       " 'namespace',\n",
       " 'll',\n",
       " 'whole',\n",
       " 'he',\n",
       " 'great',\n",
       " 'original',\n",
       " 'textview',\n",
       " 'io',\n",
       " 'hidden',\n",
       " 'internet',\n",
       " 'component',\n",
       " 'free',\n",
       " 'comment',\n",
       " 'obj',\n",
       " 'links',\n",
       " 'phone',\n",
       " 'happens',\n",
       " 'loaded',\n",
       " 'developer',\n",
       " 'messages',\n",
       " 'testing',\n",
       " 'names',\n",
       " 'automatically',\n",
       " 'whether',\n",
       " 'mvc',\n",
       " 'studio',\n",
       " '2008',\n",
       " 'play',\n",
       " 'come',\n",
       " 'course',\n",
       " 'points',\n",
       " '200',\n",
       " 'chrome',\n",
       " 'xsl',\n",
       " 'appears',\n",
       " 'related',\n",
       " 'pattern',\n",
       " 'loading',\n",
       " 'pretty',\n",
       " 'who',\n",
       " 'basic',\n",
       " 'provide',\n",
       " 'django',\n",
       " 'require',\n",
       " 'range',\n",
       " 'per',\n",
       " 'player',\n",
       " 'means',\n",
       " 'particular',\n",
       " 'padding',\n",
       " 'quite',\n",
       " 'real',\n",
       " 'tools',\n",
       " 'forms',\n",
       " 'util',\n",
       " 'datetime',\n",
       " 'buffer',\n",
       " 'character',\n",
       " 'random',\n",
       " '_post',\n",
       " 'vector',\n",
       " 'itself',\n",
       " 'ms',\n",
       " 'icon',\n",
       " 'valid',\n",
       " 'records',\n",
       " 'cursor',\n",
       " 'implementation',\n",
       " 'active',\n",
       " 'achieve',\n",
       " 'drive',\n",
       " 'yet',\n",
       " 'goes',\n",
       " 'calling',\n",
       " 'runat',\n",
       " 'max',\n",
       " 'unable',\n",
       " 'maps',\n",
       " 'bind',\n",
       " 'except',\n",
       " 'takes',\n",
       " 'params',\n",
       " 'displayed',\n",
       " 'hard',\n",
       " 'report',\n",
       " 'common',\n",
       " 'share',\n",
       " 'makes',\n",
       " 'issues',\n",
       " 'firefox',\n",
       " 'temp',\n",
       " 'runs',\n",
       " 'follows',\n",
       " 'seen',\n",
       " 'limit',\n",
       " 'properly',\n",
       " '64',\n",
       " 'company',\n",
       " 'characters',\n",
       " 'doc',\n",
       " 'applications',\n",
       " 'step',\n",
       " 'tool',\n",
       " 'native',\n",
       " 'h1',\n",
       " 'attr',\n",
       " 'sent',\n",
       " 'exist',\n",
       " 'external',\n",
       " 'duplicate',\n",
       " 'controls',\n",
       " 'complete',\n",
       " 'sum',\n",
       " 'exactly',\n",
       " 'directly',\n",
       " 'schema',\n",
       " 'panel',\n",
       " 'posts',\n",
       " 'performance',\n",
       " 'min',\n",
       " 'job',\n",
       " 'dim',\n",
       " 'difference',\n",
       " 'matrix',\n",
       " 'seconds',\n",
       " 'separate',\n",
       " 'expression',\n",
       " 'docs',\n",
       " 'syntax',\n",
       " 'textbox',\n",
       " 'examples',\n",
       " 'article',\n",
       " 'book',\n",
       " 'thinking',\n",
       " 'render',\n",
       " 'flash',\n",
       " 'mac',\n",
       " 'checkbox',\n",
       " 'printf',\n",
       " 'runtime',\n",
       " 'canvas',\n",
       " 'existing',\n",
       " 'shared',\n",
       " '28',\n",
       " 'servers',\n",
       " 'customer',\n",
       " 'desktop',\n",
       " 'buttons',\n",
       " 'previous',\n",
       " 'master',\n",
       " 'math',\n",
       " '000',\n",
       " 'comes',\n",
       " 'blog',\n",
       " 'wordpress']"
      ]
     },
     "execution_count": 24,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# An array of terms in the vocabulary\n",
    "cvModel.vocabulary"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 25,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "['customer',\n",
       " 'desktop',\n",
       " 'buttons',\n",
       " 'previous',\n",
       " 'master',\n",
       " 'math',\n",
       " '000',\n",
       " 'comes',\n",
       " 'blog',\n",
       " 'wordpress']"
      ]
     },
     "execution_count": 25,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# Show the last 10 terms in the vocabulary\n",
    "cvModel.vocabulary[-10:]"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Inter-Document Frequency"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 26,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Compute the Inverse Document Frequency (IDF) given a collection of documents\n",
    "idf = IDF(inputCol='TF', outputCol='TFIDF')\n",
    "\n",
    "# Fits a model to the input dataset with optional parameters\n",
    "idfModel = idf.fit(stack_overflow_data)\n",
    "\n",
    "# Transforms the input dataset with optional parameters\n",
    "stack_overflow_data = idfModel.transform(stack_overflow_data)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 27,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "Row(Body=\"<p>I'd like to check if an uploaded file is an image file (e.g png, jpg, jpeg, gif, bmp) or another file. The problem is that I'm using Uploadify to upload the files, which changes the mime type and gives a 'text/octal' or something as the mime type, no matter which file type you upload.</p>\\n\\n<p>Is there a way to check if the uploaded file is an image apart from checking the file extension using PHP?</p>\\n\", Id=1, Tags='php image-processing file-upload upload mime-types', Title='How to check if an uploaded file is an image without mime type?', oneTag='php', words=['p', 'i', 'd', 'like', 'to', 'check', 'if', 'an', 'uploaded', 'file', 'is', 'an', 'image', 'file', 'e', 'g', 'png', 'jpg', 'jpeg', 'gif', 'bmp', 'or', 'another', 'file', 'the', 'problem', 'is', 'that', 'i', 'm', 'using', 'uploadify', 'to', 'upload', 'the', 'files', 'which', 'changes', 'the', 'mime', 'type', 'and', 'gives', 'a', 'text', 'octal', 'or', 'something', 'as', 'the', 'mime', 'type', 'no', 'matter', 'which', 'file', 'type', 'you', 'upload', 'p', 'p', 'is', 'there', 'a', 'way', 'to', 'check', 'if', 'the', 'uploaded', 'file', 'is', 'an', 'image', 'apart', 'from', 'checking', 'the', 'file', 'extension', 'using', 'php', 'p'], BodyLength=83, NumParagraphs=2, NumLinks=0, NumFeatures=DenseVector([83.0, 2.0, 0.0]), ScaledNumFeatures=DenseVector([0.9997, 0.0241, 0.0]), ScaledNumFeatures2=DenseVector([0.4325, 0.7037, 0.0]), TF=SparseVector(1000, {0: 4.0, 1: 6.0, 2: 2.0, 3: 3.0, 5: 2.0, 8: 4.0, 9: 1.0, 15: 1.0, 21: 2.0, 28: 1.0, 31: 1.0, 35: 3.0, 36: 1.0, 43: 2.0, 45: 2.0, 48: 1.0, 51: 1.0, 57: 6.0, 61: 2.0, 71: 1.0, 78: 1.0, 84: 3.0, 86: 1.0, 94: 1.0, 97: 1.0, 99: 1.0, 100: 1.0, 115: 1.0, 147: 2.0, 152: 1.0, 169: 1.0, 241: 1.0, 283: 1.0, 306: 1.0, 350: 2.0, 490: 1.0, 578: 1.0, 759: 1.0, 832: 2.0}), TFIDF=SparseVector(1000, {0: 0.0026, 1: 0.7515, 2: 0.1374, 3: 0.3184, 5: 0.3823, 8: 1.0754, 9: 0.3344, 15: 0.5899, 21: 1.8551, 28: 1.1263, 31: 1.1113, 35: 3.3134, 36: 1.2545, 43: 2.3741, 45: 2.3753, 48: 1.2254, 51: 1.1879, 57: 11.0264, 61: 2.8957, 71: 2.1945, 78: 1.6947, 84: 6.5898, 86: 1.6136, 94: 2.3569, 97: 1.8218, 99: 2.6292, 100: 1.9206, 115: 2.3592, 147: 5.4841, 152: 2.1116, 169: 2.6328, 241: 2.5745, 283: 3.2325, 306: 3.2668, 350: 6.2367, 490: 3.8893, 578: 3.6182, 759: 3.7771, 832: 8.8964}))"
      ]
     },
     "execution_count": 27,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# Returns the first ``n`` rows\n",
    "stack_overflow_data.head()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### StringIndexer"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 28,
   "metadata": {},
   "outputs": [],
   "source": [
    "# A label indexer that maps a string column of labels to an ML column of label indices\n",
    "# If the input column is numeric, we cast it to string and index the string values\n",
    "# The indices are in [0, numLabels). By default, this is ordered by label frequencies so the most frequent label gets index 0\n",
    "# Note: stringOrderType: the ordering behavior; default value is 'frequencyDesc'; other option is 'alphabetDesc'\n",
    "stringIndexer = StringIndexer(inputCol='oneTag', outputCol='label')\n",
    "\n",
    "# Fits a model to the input dataset with optional parameters\n",
    "stringIndexerModel = stringIndexer.fit(stack_overflow_data)\n",
    "\n",
    "# Transforms the input dataset with optional parameters\n",
    "stack_overflow_data = stringIndexerModel.transform(stack_overflow_data)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 29,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "Row(Body=\"<p>I'd like to check if an uploaded file is an image file (e.g png, jpg, jpeg, gif, bmp) or another file. The problem is that I'm using Uploadify to upload the files, which changes the mime type and gives a 'text/octal' or something as the mime type, no matter which file type you upload.</p>\\n\\n<p>Is there a way to check if the uploaded file is an image apart from checking the file extension using PHP?</p>\\n\", Id=1, Tags='php image-processing file-upload upload mime-types', Title='How to check if an uploaded file is an image without mime type?', oneTag='php', words=['p', 'i', 'd', 'like', 'to', 'check', 'if', 'an', 'uploaded', 'file', 'is', 'an', 'image', 'file', 'e', 'g', 'png', 'jpg', 'jpeg', 'gif', 'bmp', 'or', 'another', 'file', 'the', 'problem', 'is', 'that', 'i', 'm', 'using', 'uploadify', 'to', 'upload', 'the', 'files', 'which', 'changes', 'the', 'mime', 'type', 'and', 'gives', 'a', 'text', 'octal', 'or', 'something', 'as', 'the', 'mime', 'type', 'no', 'matter', 'which', 'file', 'type', 'you', 'upload', 'p', 'p', 'is', 'there', 'a', 'way', 'to', 'check', 'if', 'the', 'uploaded', 'file', 'is', 'an', 'image', 'apart', 'from', 'checking', 'the', 'file', 'extension', 'using', 'php', 'p'], BodyLength=83, NumParagraphs=2, NumLinks=0, NumFeatures=DenseVector([83.0, 2.0, 0.0]), ScaledNumFeatures=DenseVector([0.9997, 0.0241, 0.0]), ScaledNumFeatures2=DenseVector([0.4325, 0.7037, 0.0]), TF=SparseVector(1000, {0: 4.0, 1: 6.0, 2: 2.0, 3: 3.0, 5: 2.0, 8: 4.0, 9: 1.0, 15: 1.0, 21: 2.0, 28: 1.0, 31: 1.0, 35: 3.0, 36: 1.0, 43: 2.0, 45: 2.0, 48: 1.0, 51: 1.0, 57: 6.0, 61: 2.0, 71: 1.0, 78: 1.0, 84: 3.0, 86: 1.0, 94: 1.0, 97: 1.0, 99: 1.0, 100: 1.0, 115: 1.0, 147: 2.0, 152: 1.0, 169: 1.0, 241: 1.0, 283: 1.0, 306: 1.0, 350: 2.0, 490: 1.0, 578: 1.0, 759: 1.0, 832: 2.0}), TFIDF=SparseVector(1000, {0: 0.0026, 1: 0.7515, 2: 0.1374, 3: 0.3184, 5: 0.3823, 8: 1.0754, 9: 0.3344, 15: 0.5899, 21: 1.8551, 28: 1.1263, 31: 1.1113, 35: 3.3134, 36: 1.2545, 43: 2.3741, 45: 2.3753, 48: 1.2254, 51: 1.1879, 57: 11.0264, 61: 2.8957, 71: 2.1945, 78: 1.6947, 84: 6.5898, 86: 1.6136, 94: 2.3569, 97: 1.8218, 99: 2.6292, 100: 1.9206, 115: 2.3592, 147: 5.4841, 152: 2.1116, 169: 2.6328, 241: 2.5745, 283: 3.2325, 306: 3.2668, 350: 6.2367, 490: 3.8893, 578: 3.6182, 759: 3.7771, 832: 8.8964}), label=3.0)"
      ]
     },
     "execution_count": 29,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# Returns the first ``n`` rows\n",
    "stack_overflow_data.head()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Creating Features"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Read in the Data Set"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 30,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "DataFrame[Body: string, Id: bigint, Tags: string, Title: string, oneTag: string]"
      ]
     },
     "execution_count": 30,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "path = '/Users/yangweichle/Documents/Employment/TRAINING/DATA SCIENCE/Spark/Udacity_Spark for Big Data/Machine Learning with Spark/data/Train_onetag_small.json'\n",
    "\n",
    "# Loads JSON files and returns the results as a `DataFrame`\n",
    "# Note: path: string represents path to the JSON dataset, or a list of paths, or RDD of Strings storing JSON objects\n",
    "stack_overflow_data2 = spark.read.json(path=path)\n",
    "\n",
    "# Sets the storage level to persist the contents of the `DataFrame` across operations after the first time it is computed\n",
    "# This can only be used to assign a new storage level if the `DataFrame` does not have a storage level set yet\n",
    "# If no storage level is specified defaults to (C{MEMORY_AND_DISK})\n",
    "stack_overflow_data2.persist()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 31,
   "metadata": {},
   "outputs": [],
   "source": [
    "# A regex based tokenizer that extracts tokens either by using the provided regex pattern (in Java dialect) to \n",
    "#    split the text (default) or repeatedly matching the regex (if gaps is false)\n",
    "# Optional parameters also allow filtering tokens using a minimal length\n",
    "# It returns an array of strings that can be empty \n",
    "regexTokenizer = RegexTokenizer(inputCol='Body', outputCol='words', pattern='\\\\W')\n",
    "\n",
    "# Transforms the input dataset with optional parameters\n",
    "stack_overflow_data2 = regexTokenizer.transform(stack_overflow_data2)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 32,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Create a user defined function (UDF)\n",
    "body_length = udf(lambda x: len(x), IntegerType())\n",
    "\n",
    "# Returns a new `DataFrame` by adding a column or replacing the existing column that has the same name\n",
    "stack_overflow_data2 = stack_overflow_data2.withColumn('BodyLength', body_length(stack_overflow_data2.words))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 33,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "Row(Body=\"<p>I'd like to check if an uploaded file is an image file (e.g png, jpg, jpeg, gif, bmp) or another file. The problem is that I'm using Uploadify to upload the files, which changes the mime type and gives a 'text/octal' or something as the mime type, no matter which file type you upload.</p>\\n\\n<p>Is there a way to check if the uploaded file is an image apart from checking the file extension using PHP?</p>\\n\", Id=1, Tags='php image-processing file-upload upload mime-types', Title='How to check if an uploaded file is an image without mime type?', oneTag='php', words=['p', 'i', 'd', 'like', 'to', 'check', 'if', 'an', 'uploaded', 'file', 'is', 'an', 'image', 'file', 'e', 'g', 'png', 'jpg', 'jpeg', 'gif', 'bmp', 'or', 'another', 'file', 'the', 'problem', 'is', 'that', 'i', 'm', 'using', 'uploadify', 'to', 'upload', 'the', 'files', 'which', 'changes', 'the', 'mime', 'type', 'and', 'gives', 'a', 'text', 'octal', 'or', 'something', 'as', 'the', 'mime', 'type', 'no', 'matter', 'which', 'file', 'type', 'you', 'upload', 'p', 'p', 'is', 'there', 'a', 'way', 'to', 'check', 'if', 'the', 'uploaded', 'file', 'is', 'an', 'image', 'apart', 'from', 'checking', 'the', 'file', 'extension', 'using', 'php', 'p'], BodyLength=83)"
      ]
     },
     "execution_count": 33,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# Returns the first ``n`` rows\n",
    "stack_overflow_data2.head()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Question 1\n",
    "\n",
    "Select the question with Id = 1112. How many words does its body contain (check the `BodyLength` column)?"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 34,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "+--------------------+----+--------------------+--------------------+------+--------------------+----------+\n",
      "|                Body|  Id|                Tags|               Title|oneTag|               words|BodyLength|\n",
      "+--------------------+----+--------------------+--------------------+------+--------------------+----------+\n",
      "|<p>I submitted my...|1112|iphone app-store ...|iPhone app releas...|iphone|[p, i, submitted,...|        63|\n",
      "+--------------------+----+--------------------+--------------------+------+--------------------+----------+\n",
      "\n"
     ]
    }
   ],
   "source": [
    "# Filters rows using the given condition\n",
    "stack_overflow_data2.where(stack_overflow_data2.Id == 1112).show()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Question 2\n",
    "\n",
    "Create a new column that concatenates the question `Title` and `Body`. Apply the same functions we used before to compute the number of words in this combined column. What's the value in this new column for Id = 5123?"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 35,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Returns a new `DataFrame` by adding a column or replacing the existing column that has the same name\n",
    "# concat: concatenates multiple input columns together into a single column;\n",
    "#         the function works with strings, binary and compatible array columns\n",
    "stack_overflow_data2 = stack_overflow_data2.withColumn('Desc', concat(col('Title'), lit(' '), col('Body')))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 36,
   "metadata": {},
   "outputs": [],
   "source": [
    "# A regex based tokenizer that extracts tokens either by using the provided regex pattern (in Java dialect) to \n",
    "#    split the text (default) or repeatedly matching the regex (if gaps is false)\n",
    "# Optional parameters also allow filtering tokens using a minimal length\n",
    "# It returns an array of strings that can be empty \n",
    "regexTokenizer = RegexTokenizer(inputCol='Desc', outputCol='words2', pattern='\\\\W')\n",
    "\n",
    "# Transforms the input dataset with optional parameters\n",
    "stack_overflow_data2 = regexTokenizer.transform(stack_overflow_data2)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 37,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Returns a new `DataFrame` by adding a column or replacing the existing column that has the same name\n",
    "stack_overflow_data2 = stack_overflow_data2.withColumn('DescLength', body_length(stack_overflow_data2.words2))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 38,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "[Row(Body=\"<p>Here's an interesting experiment with using Git. Think of Github's ‘pages’ feature: I write a program in one branch (e.g. <code>master</code>), and a documentation website is kept in another, entirely unrelated branch (e.g. <code>gh-pages</code>).</p>\\n\\n<p>I can generate documentation in HTML format from the code in my <code>master</code>-branch, but I want to publish this as part of my documentation website in the <code>gh-pages</code> branch.</p>\\n\\n<p>How could I intelligently generate my docs from my code in <code>master</code>, move it to my <code>gh-pages</code> branch and commit the changes there? Should I use a post-commit hook or something? Would this be a good idea, or is it utterly foolish?</p>\\n\", Id=5123, Tags='git branch', Title='Git branch experiment', oneTag='git', words=['p', 'here', 's', 'an', 'interesting', 'experiment', 'with', 'using', 'git', 'think', 'of', 'github', 's', 'pages', 'feature', 'i', 'write', 'a', 'program', 'in', 'one', 'branch', 'e', 'g', 'code', 'master', 'code', 'and', 'a', 'documentation', 'website', 'is', 'kept', 'in', 'another', 'entirely', 'unrelated', 'branch', 'e', 'g', 'code', 'gh', 'pages', 'code', 'p', 'p', 'i', 'can', 'generate', 'documentation', 'in', 'html', 'format', 'from', 'the', 'code', 'in', 'my', 'code', 'master', 'code', 'branch', 'but', 'i', 'want', 'to', 'publish', 'this', 'as', 'part', 'of', 'my', 'documentation', 'website', 'in', 'the', 'code', 'gh', 'pages', 'code', 'branch', 'p', 'p', 'how', 'could', 'i', 'intelligently', 'generate', 'my', 'docs', 'from', 'my', 'code', 'in', 'code', 'master', 'code', 'move', 'it', 'to', 'my', 'code', 'gh', 'pages', 'code', 'branch', 'and', 'commit', 'the', 'changes', 'there', 'should', 'i', 'use', 'a', 'post', 'commit', 'hook', 'or', 'something', 'would', 'this', 'be', 'a', 'good', 'idea', 'or', 'is', 'it', 'utterly', 'foolish', 'p'], BodyLength=132, Desc=\"Git branch experiment <p>Here's an interesting experiment with using Git. Think of Github's ‘pages’ feature: I write a program in one branch (e.g. <code>master</code>), and a documentation website is kept in another, entirely unrelated branch (e.g. <code>gh-pages</code>).</p>\\n\\n<p>I can generate documentation in HTML format from the code in my <code>master</code>-branch, but I want to publish this as part of my documentation website in the <code>gh-pages</code> branch.</p>\\n\\n<p>How could I intelligently generate my docs from my code in <code>master</code>, move it to my <code>gh-pages</code> branch and commit the changes there? Should I use a post-commit hook or something? Would this be a good idea, or is it utterly foolish?</p>\\n\", words2=['git', 'branch', 'experiment', 'p', 'here', 's', 'an', 'interesting', 'experiment', 'with', 'using', 'git', 'think', 'of', 'github', 's', 'pages', 'feature', 'i', 'write', 'a', 'program', 'in', 'one', 'branch', 'e', 'g', 'code', 'master', 'code', 'and', 'a', 'documentation', 'website', 'is', 'kept', 'in', 'another', 'entirely', 'unrelated', 'branch', 'e', 'g', 'code', 'gh', 'pages', 'code', 'p', 'p', 'i', 'can', 'generate', 'documentation', 'in', 'html', 'format', 'from', 'the', 'code', 'in', 'my', 'code', 'master', 'code', 'branch', 'but', 'i', 'want', 'to', 'publish', 'this', 'as', 'part', 'of', 'my', 'documentation', 'website', 'in', 'the', 'code', 'gh', 'pages', 'code', 'branch', 'p', 'p', 'how', 'could', 'i', 'intelligently', 'generate', 'my', 'docs', 'from', 'my', 'code', 'in', 'code', 'master', 'code', 'move', 'it', 'to', 'my', 'code', 'gh', 'pages', 'code', 'branch', 'and', 'commit', 'the', 'changes', 'there', 'should', 'i', 'use', 'a', 'post', 'commit', 'hook', 'or', 'something', 'would', 'this', 'be', 'a', 'good', 'idea', 'or', 'is', 'it', 'utterly', 'foolish', 'p'], DescLength=135)]"
      ]
     },
     "execution_count": 38,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# Filters rows using the given condition\n",
    "stack_overflow_data2.where(stack_overflow_data2.Id == 5123).collect()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Create a Vector\n",
    "\n",
    "Create a vector from the combined Title + Body length column. In the next few questions, you'll try different normalizer/scaler methods on this new column."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 39,
   "metadata": {},
   "outputs": [],
   "source": [
    "# A feature transformer that merges multiple columns into a vector column\n",
    "vecAssembler = VectorAssembler(inputCols=['DescLength'], outputCol='DescVec')\n",
    "\n",
    "# Transforms the input dataset with optional parameters\n",
    "stack_overflow_data2 = vecAssembler.transform(stack_overflow_data2)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Question 3\n",
    "\n",
    "Using the `Normalizer` method, what's the normalized value for question Id = 512?"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 40,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Normalize a vector to have unit norm using the given p-norm\n",
    "normalizer = Normalizer(inputCol='DescVec', outputCol='DescVecNormalizer')\n",
    "\n",
    "# Transforms the input dataset with optional parameters\n",
    "stack_overflow_data2 = normalizer.transform(stack_overflow_data2)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 41,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "[Row(Body=\"<p>I'd like to have a better understanding of what optimizations HotSpot might generate for my Java code at run time. </p>\\n\\n<p>Is there a way to see the optimized code that HotSpot is using after it's been running for a while?</p>\\n\", Id=512, Tags='java optimization hotspot', Title='How can I see the code that HotSpot generates after optimizing?', oneTag='java', words=['p', 'i', 'd', 'like', 'to', 'have', 'a', 'better', 'understanding', 'of', 'what', 'optimizations', 'hotspot', 'might', 'generate', 'for', 'my', 'java', 'code', 'at', 'run', 'time', 'p', 'p', 'is', 'there', 'a', 'way', 'to', 'see', 'the', 'optimized', 'code', 'that', 'hotspot', 'is', 'using', 'after', 'it', 's', 'been', 'running', 'for', 'a', 'while', 'p'], BodyLength=46, Desc=\"How can I see the code that HotSpot generates after optimizing? <p>I'd like to have a better understanding of what optimizations HotSpot might generate for my Java code at run time. </p>\\n\\n<p>Is there a way to see the optimized code that HotSpot is using after it's been running for a while?</p>\\n\", words2=['how', 'can', 'i', 'see', 'the', 'code', 'that', 'hotspot', 'generates', 'after', 'optimizing', 'p', 'i', 'd', 'like', 'to', 'have', 'a', 'better', 'understanding', 'of', 'what', 'optimizations', 'hotspot', 'might', 'generate', 'for', 'my', 'java', 'code', 'at', 'run', 'time', 'p', 'p', 'is', 'there', 'a', 'way', 'to', 'see', 'the', 'optimized', 'code', 'that', 'hotspot', 'is', 'using', 'after', 'it', 's', 'been', 'running', 'for', 'a', 'while', 'p'], DescLength=57, DescVec=DenseVector([57.0]), DescVecNormalizer=DenseVector([1.0]))]"
      ]
     },
     "execution_count": 41,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# Filters rows using the given condition\n",
    "stack_overflow_data2.where(stack_overflow_data2.Id == 512).collect()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Question 4\n",
    "\n",
    "Using the `StandardScaler` method (scaling both the mean and the standard deviation) what's the normalized value for question Id = 512?"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 42,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Standardizes features by removing the mean and scaling to unit variance using column  summary statistics \n",
    "#    on the samples in the training set\n",
    "# The \"unit std\" is computed using the `corrected sample standard deviation`, which is computed as the \n",
    "#    square root of the unbiased sample variance\n",
    "standardScaler = StandardScaler(inputCol='DescVec', outputCol='DescVecStandardScaler', withMean=True, withStd=True)\n",
    "\n",
    "# Fits a model to the input dataset with optional parameters\n",
    "scalerModel = standardScaler.fit(stack_overflow_data2)\n",
    "\n",
    "# Transforms the input dataset with optional parameters\n",
    "stack_overflow_data2 = scalerModel.transform(stack_overflow_data2)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 43,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "[Row(Body=\"<p>I'd like to have a better understanding of what optimizations HotSpot might generate for my Java code at run time. </p>\\n\\n<p>Is there a way to see the optimized code that HotSpot is using after it's been running for a while?</p>\\n\", Id=512, Tags='java optimization hotspot', Title='How can I see the code that HotSpot generates after optimizing?', oneTag='java', words=['p', 'i', 'd', 'like', 'to', 'have', 'a', 'better', 'understanding', 'of', 'what', 'optimizations', 'hotspot', 'might', 'generate', 'for', 'my', 'java', 'code', 'at', 'run', 'time', 'p', 'p', 'is', 'there', 'a', 'way', 'to', 'see', 'the', 'optimized', 'code', 'that', 'hotspot', 'is', 'using', 'after', 'it', 's', 'been', 'running', 'for', 'a', 'while', 'p'], BodyLength=46, Desc=\"How can I see the code that HotSpot generates after optimizing? <p>I'd like to have a better understanding of what optimizations HotSpot might generate for my Java code at run time. </p>\\n\\n<p>Is there a way to see the optimized code that HotSpot is using after it's been running for a while?</p>\\n\", words2=['how', 'can', 'i', 'see', 'the', 'code', 'that', 'hotspot', 'generates', 'after', 'optimizing', 'p', 'i', 'd', 'like', 'to', 'have', 'a', 'better', 'understanding', 'of', 'what', 'optimizations', 'hotspot', 'might', 'generate', 'for', 'my', 'java', 'code', 'at', 'run', 'time', 'p', 'p', 'is', 'there', 'a', 'way', 'to', 'see', 'the', 'optimized', 'code', 'that', 'hotspot', 'is', 'using', 'after', 'it', 's', 'been', 'running', 'for', 'a', 'while', 'p'], DescLength=57, DescVec=DenseVector([57.0]), DescVecNormalizer=DenseVector([1.0]), DescVecStandardScaler=DenseVector([-0.6417]))]"
      ]
     },
     "execution_count": 43,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# Filters rows using the given condition\n",
    "stack_overflow_data2.where(stack_overflow_data2.Id == 512).collect()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Question 5\n",
    "\n",
    "Using the `MinMAxScaler` method, what's the normalized value for question Id = 512?"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 44,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Rescale each feature individually to a common range [min, max] linearly using column summary statistics, \n",
    "#   which is also known as min-max normalization or Rescaling. The rescaled value for feature E is calculated as,\n",
    "#   Rescaled(e_i) = (e_i - E_min) / (E_max - E_min) * (max - min) + min\n",
    "# For the case E_max == E_min, Rescaled(e_i) = 0.5 * (max + min)\n",
    "mmScaler = MinMaxScaler(inputCol='DescVec', outputCol='DescVecMinMaxScaler')\n",
    "\n",
    "# Fits a model to the input dataset with optional parameters\n",
    "mmScalerModel = mmScaler.fit(stack_overflow_data2)\n",
    "\n",
    "# Transforms the input dataset with optional parameters\n",
    "stack_overflow_data2 = mmScalerModel.transform(stack_overflow_data2)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 45,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "[Row(Body=\"<p>I'd like to have a better understanding of what optimizations HotSpot might generate for my Java code at run time. </p>\\n\\n<p>Is there a way to see the optimized code that HotSpot is using after it's been running for a while?</p>\\n\", Id=512, Tags='java optimization hotspot', Title='How can I see the code that HotSpot generates after optimizing?', oneTag='java', words=['p', 'i', 'd', 'like', 'to', 'have', 'a', 'better', 'understanding', 'of', 'what', 'optimizations', 'hotspot', 'might', 'generate', 'for', 'my', 'java', 'code', 'at', 'run', 'time', 'p', 'p', 'is', 'there', 'a', 'way', 'to', 'see', 'the', 'optimized', 'code', 'that', 'hotspot', 'is', 'using', 'after', 'it', 's', 'been', 'running', 'for', 'a', 'while', 'p'], BodyLength=46, Desc=\"How can I see the code that HotSpot generates after optimizing? <p>I'd like to have a better understanding of what optimizations HotSpot might generate for my Java code at run time. </p>\\n\\n<p>Is there a way to see the optimized code that HotSpot is using after it's been running for a while?</p>\\n\", words2=['how', 'can', 'i', 'see', 'the', 'code', 'that', 'hotspot', 'generates', 'after', 'optimizing', 'p', 'i', 'd', 'like', 'to', 'have', 'a', 'better', 'understanding', 'of', 'what', 'optimizations', 'hotspot', 'might', 'generate', 'for', 'my', 'java', 'code', 'at', 'run', 'time', 'p', 'p', 'is', 'there', 'a', 'way', 'to', 'see', 'the', 'optimized', 'code', 'that', 'hotspot', 'is', 'using', 'after', 'it', 's', 'been', 'running', 'for', 'a', 'while', 'p'], DescLength=57, DescVec=DenseVector([57.0]), DescVecNormalizer=DenseVector([1.0]), DescVecStandardScaler=DenseVector([-0.6417]), DescVecMinMaxScaler=DenseVector([0.0062]))]"
      ]
     },
     "execution_count": 45,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# Filters rows using the given condition\n",
    "stack_overflow_data2.where(stack_overflow_data2.Id == 512).collect()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Dimensionality Reduction"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 46,
   "metadata": {},
   "outputs": [],
   "source": [
    "# PCA trains a model to project vectors to a lower dimensional space of the top `k` principal components\n",
    "pca = PCA(k=100, inputCol='TFIDF', outputCol='pcaTFIDF')\n",
    "\n",
    "# Fits a model to the input dataset with optional parameters\n",
    "pcaModel = pca.fit(stack_overflow_data)\n",
    "\n",
    "# Transforms the input dataset with optional parameters\n",
    "stack_overflow_data = pcaModel.transform(stack_overflow_data)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 47,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "Row(Body=\"<p>I'd like to check if an uploaded file is an image file (e.g png, jpg, jpeg, gif, bmp) or another file. The problem is that I'm using Uploadify to upload the files, which changes the mime type and gives a 'text/octal' or something as the mime type, no matter which file type you upload.</p>\\n\\n<p>Is there a way to check if the uploaded file is an image apart from checking the file extension using PHP?</p>\\n\", Id=1, Tags='php image-processing file-upload upload mime-types', Title='How to check if an uploaded file is an image without mime type?', oneTag='php', words=['p', 'i', 'd', 'like', 'to', 'check', 'if', 'an', 'uploaded', 'file', 'is', 'an', 'image', 'file', 'e', 'g', 'png', 'jpg', 'jpeg', 'gif', 'bmp', 'or', 'another', 'file', 'the', 'problem', 'is', 'that', 'i', 'm', 'using', 'uploadify', 'to', 'upload', 'the', 'files', 'which', 'changes', 'the', 'mime', 'type', 'and', 'gives', 'a', 'text', 'octal', 'or', 'something', 'as', 'the', 'mime', 'type', 'no', 'matter', 'which', 'file', 'type', 'you', 'upload', 'p', 'p', 'is', 'there', 'a', 'way', 'to', 'check', 'if', 'the', 'uploaded', 'file', 'is', 'an', 'image', 'apart', 'from', 'checking', 'the', 'file', 'extension', 'using', 'php', 'p'], BodyLength=83, NumParagraphs=2, NumLinks=0, NumFeatures=DenseVector([83.0, 2.0, 0.0]), ScaledNumFeatures=DenseVector([0.9997, 0.0241, 0.0]), ScaledNumFeatures2=DenseVector([0.4325, 0.7037, 0.0]), TF=SparseVector(1000, {0: 4.0, 1: 6.0, 2: 2.0, 3: 3.0, 5: 2.0, 8: 4.0, 9: 1.0, 15: 1.0, 21: 2.0, 28: 1.0, 31: 1.0, 35: 3.0, 36: 1.0, 43: 2.0, 45: 2.0, 48: 1.0, 51: 1.0, 57: 6.0, 61: 2.0, 71: 1.0, 78: 1.0, 84: 3.0, 86: 1.0, 94: 1.0, 97: 1.0, 99: 1.0, 100: 1.0, 115: 1.0, 147: 2.0, 152: 1.0, 169: 1.0, 241: 1.0, 283: 1.0, 306: 1.0, 350: 2.0, 490: 1.0, 578: 1.0, 759: 1.0, 832: 2.0}), TFIDF=SparseVector(1000, {0: 0.0026, 1: 0.7515, 2: 0.1374, 3: 0.3184, 5: 0.3823, 8: 1.0754, 9: 0.3344, 15: 0.5899, 21: 1.8551, 28: 1.1263, 31: 1.1113, 35: 3.3134, 36: 1.2545, 43: 2.3741, 45: 2.3753, 48: 1.2254, 51: 1.1879, 57: 11.0264, 61: 2.8957, 71: 2.1945, 78: 1.6947, 84: 6.5898, 86: 1.6136, 94: 2.3569, 97: 1.8218, 99: 2.6292, 100: 1.9206, 115: 2.3592, 147: 5.4841, 152: 2.1116, 169: 2.6328, 241: 2.5745, 283: 3.2325, 306: 3.2668, 350: 6.2367, 490: 3.8893, 578: 3.6182, 759: 3.7771, 832: 8.8964}), label=3.0, pcaTFIDF=DenseVector([-0.5291, -0.8217, 0.3129, 0.02, 0.0323, -0.1371, 0.1032, -0.1511, 0.4816, -0.2657, 0.9031, -0.1125, 3.1339, -0.4338, -0.1165, 0.3052, 0.9695, -0.7508, 0.1898, -1.0876, 0.5371, -0.8804, 1.5681, -0.3721, -0.4511, 0.6415, 0.5597, -0.0773, 0.4399, 1.0323, -0.8446, 0.7257, -0.6349, -1.3363, -0.9206, 1.5778, -1.8451, -0.2224, -1.1524, -0.0381, -0.0415, 0.3505, 1.2341, -0.4662, 0.8383, 0.772, 0.7149, -1.0151, 0.148, 0.1278, -0.946, -0.6953, -1.5553, -0.9866, 0.7846, -0.7185, 0.946, 0.6609, -0.0182, 1.3281, -0.4261, -0.6093, -0.8237, -0.5232, -0.5305, -0.4872, 0.1315, 0.8463, -1.1532, -1.2489, 0.3981, -1.4053, 0.4366, -0.931, 0.062, 0.9369, 0.8366, -0.7272, 1.5533, -1.9902, -0.4451, 0.9578, 0.364, -0.3055, -0.9719, -1.1939, 1.1266, -0.3546, 1.6776, 2.1847, -0.0966, -1.6945, -0.9625, -0.7207, 0.4287, -0.6703, 0.7134, 0.2583, -1.692, -0.4525]))"
      ]
     },
     "execution_count": 47,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# Returns the first ``n`` rows\n",
    "stack_overflow_data.head()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Supervised ML Algorithms"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Linear Regression"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 48,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Create a user defined function (UDF)\n",
    "number_of_tags = udf(lambda x: len(x.split(' ')), IntegerType())\n",
    "\n",
    "# Returns a new `DataFrame` by adding a column or replacing the existing column that has the same name\n",
    "stack_overflow_data = stack_overflow_data.withColumn('NumTags', number_of_tags(stack_overflow_data.Tags))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 49,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "Row(Body=\"<p>I'd like to check if an uploaded file is an image file (e.g png, jpg, jpeg, gif, bmp) or another file. The problem is that I'm using Uploadify to upload the files, which changes the mime type and gives a 'text/octal' or something as the mime type, no matter which file type you upload.</p>\\n\\n<p>Is there a way to check if the uploaded file is an image apart from checking the file extension using PHP?</p>\\n\", Id=1, Tags='php image-processing file-upload upload mime-types', Title='How to check if an uploaded file is an image without mime type?', oneTag='php', words=['p', 'i', 'd', 'like', 'to', 'check', 'if', 'an', 'uploaded', 'file', 'is', 'an', 'image', 'file', 'e', 'g', 'png', 'jpg', 'jpeg', 'gif', 'bmp', 'or', 'another', 'file', 'the', 'problem', 'is', 'that', 'i', 'm', 'using', 'uploadify', 'to', 'upload', 'the', 'files', 'which', 'changes', 'the', 'mime', 'type', 'and', 'gives', 'a', 'text', 'octal', 'or', 'something', 'as', 'the', 'mime', 'type', 'no', 'matter', 'which', 'file', 'type', 'you', 'upload', 'p', 'p', 'is', 'there', 'a', 'way', 'to', 'check', 'if', 'the', 'uploaded', 'file', 'is', 'an', 'image', 'apart', 'from', 'checking', 'the', 'file', 'extension', 'using', 'php', 'p'], BodyLength=83, NumParagraphs=2, NumLinks=0, NumFeatures=DenseVector([83.0, 2.0, 0.0]), ScaledNumFeatures=DenseVector([0.9997, 0.0241, 0.0]), ScaledNumFeatures2=DenseVector([0.4325, 0.7037, 0.0]), TF=SparseVector(1000, {0: 4.0, 1: 6.0, 2: 2.0, 3: 3.0, 5: 2.0, 8: 4.0, 9: 1.0, 15: 1.0, 21: 2.0, 28: 1.0, 31: 1.0, 35: 3.0, 36: 1.0, 43: 2.0, 45: 2.0, 48: 1.0, 51: 1.0, 57: 6.0, 61: 2.0, 71: 1.0, 78: 1.0, 84: 3.0, 86: 1.0, 94: 1.0, 97: 1.0, 99: 1.0, 100: 1.0, 115: 1.0, 147: 2.0, 152: 1.0, 169: 1.0, 241: 1.0, 283: 1.0, 306: 1.0, 350: 2.0, 490: 1.0, 578: 1.0, 759: 1.0, 832: 2.0}), TFIDF=SparseVector(1000, {0: 0.0026, 1: 0.7515, 2: 0.1374, 3: 0.3184, 5: 0.3823, 8: 1.0754, 9: 0.3344, 15: 0.5899, 21: 1.8551, 28: 1.1263, 31: 1.1113, 35: 3.3134, 36: 1.2545, 43: 2.3741, 45: 2.3753, 48: 1.2254, 51: 1.1879, 57: 11.0264, 61: 2.8957, 71: 2.1945, 78: 1.6947, 84: 6.5898, 86: 1.6136, 94: 2.3569, 97: 1.8218, 99: 2.6292, 100: 1.9206, 115: 2.3592, 147: 5.4841, 152: 2.1116, 169: 2.6328, 241: 2.5745, 283: 3.2325, 306: 3.2668, 350: 6.2367, 490: 3.8893, 578: 3.6182, 759: 3.7771, 832: 8.8964}), label=3.0, pcaTFIDF=DenseVector([-0.5291, -0.8217, 0.3129, 0.02, 0.0323, -0.1371, 0.1032, -0.1511, 0.4816, -0.2657, 0.9031, -0.1125, 3.1339, -0.4338, -0.1165, 0.3052, 0.9695, -0.7508, 0.1898, -1.0876, 0.5371, -0.8804, 1.5681, -0.3721, -0.4511, 0.6415, 0.5597, -0.0773, 0.4399, 1.0323, -0.8446, 0.7257, -0.6349, -1.3363, -0.9206, 1.5778, -1.8451, -0.2224, -1.1524, -0.0381, -0.0415, 0.3505, 1.2341, -0.4662, 0.8383, 0.772, 0.7149, -1.0151, 0.148, 0.1278, -0.946, -0.6953, -1.5553, -0.9866, 0.7846, -0.7185, 0.946, 0.6609, -0.0182, 1.3281, -0.4261, -0.6093, -0.8237, -0.5232, -0.5305, -0.4872, 0.1315, 0.8463, -1.1532, -1.2489, 0.3981, -1.4053, 0.4366, -0.931, 0.062, 0.9369, 0.8366, -0.7272, 1.5533, -1.9902, -0.4451, 0.9578, 0.364, -0.3055, -0.9719, -1.1939, 1.1266, -0.3546, 1.6776, 2.1847, -0.0966, -1.6945, -0.9625, -0.7207, 0.4287, -0.6703, 0.7134, 0.2583, -1.692, -0.4525]), NumTags=5)"
      ]
     },
     "execution_count": 49,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# Returns the first ``n`` rows\n",
    "stack_overflow_data.head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 50,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "+-------+-----+\n",
      "|NumTags|count|\n",
      "+-------+-----+\n",
      "|      1|13858|\n",
      "|      2|26540|\n",
      "|      3|28769|\n",
      "|      4|19108|\n",
      "|      5|11725|\n",
      "+-------+-----+\n",
      "\n"
     ]
    }
   ],
   "source": [
    "stack_overflow_data.groupby('NumTags').count().orderBy('NumTags').show()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 51,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "+-------+------------------+\n",
      "|NumTags|   avg(BodyLength)|\n",
      "+-------+------------------+\n",
      "|      1|135.41311877615817|\n",
      "|      2|153.82456669178598|\n",
      "|      3|172.73704334526747|\n",
      "|      4|192.67050450073268|\n",
      "|      5|218.54251599147122|\n",
      "+-------+------------------+\n",
      "\n"
     ]
    }
   ],
   "source": [
    "stack_overflow_data.groupby('NumTags').agg(avg(col('BodyLength'))).orderBy('NumTags').show()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 52,
   "metadata": {},
   "outputs": [],
   "source": [
    "# A feature transformer that merges multiple columns into a vector column\n",
    "vecAssembler = VectorAssembler(inputCols=['BodyLength'], outputCol='LengthFeature')\n",
    "\n",
    "# Transforms the input dataset with optional parameters\n",
    "stack_overflow_data = vecAssembler.transform(stack_overflow_data)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 53,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "Row(Body=\"<p>I'd like to check if an uploaded file is an image file (e.g png, jpg, jpeg, gif, bmp) or another file. The problem is that I'm using Uploadify to upload the files, which changes the mime type and gives a 'text/octal' or something as the mime type, no matter which file type you upload.</p>\\n\\n<p>Is there a way to check if the uploaded file is an image apart from checking the file extension using PHP?</p>\\n\", Id=1, Tags='php image-processing file-upload upload mime-types', Title='How to check if an uploaded file is an image without mime type?', oneTag='php', words=['p', 'i', 'd', 'like', 'to', 'check', 'if', 'an', 'uploaded', 'file', 'is', 'an', 'image', 'file', 'e', 'g', 'png', 'jpg', 'jpeg', 'gif', 'bmp', 'or', 'another', 'file', 'the', 'problem', 'is', 'that', 'i', 'm', 'using', 'uploadify', 'to', 'upload', 'the', 'files', 'which', 'changes', 'the', 'mime', 'type', 'and', 'gives', 'a', 'text', 'octal', 'or', 'something', 'as', 'the', 'mime', 'type', 'no', 'matter', 'which', 'file', 'type', 'you', 'upload', 'p', 'p', 'is', 'there', 'a', 'way', 'to', 'check', 'if', 'the', 'uploaded', 'file', 'is', 'an', 'image', 'apart', 'from', 'checking', 'the', 'file', 'extension', 'using', 'php', 'p'], BodyLength=83, NumParagraphs=2, NumLinks=0, NumFeatures=DenseVector([83.0, 2.0, 0.0]), ScaledNumFeatures=DenseVector([0.9997, 0.0241, 0.0]), ScaledNumFeatures2=DenseVector([0.4325, 0.7037, 0.0]), TF=SparseVector(1000, {0: 4.0, 1: 6.0, 2: 2.0, 3: 3.0, 5: 2.0, 8: 4.0, 9: 1.0, 15: 1.0, 21: 2.0, 28: 1.0, 31: 1.0, 35: 3.0, 36: 1.0, 43: 2.0, 45: 2.0, 48: 1.0, 51: 1.0, 57: 6.0, 61: 2.0, 71: 1.0, 78: 1.0, 84: 3.0, 86: 1.0, 94: 1.0, 97: 1.0, 99: 1.0, 100: 1.0, 115: 1.0, 147: 2.0, 152: 1.0, 169: 1.0, 241: 1.0, 283: 1.0, 306: 1.0, 350: 2.0, 490: 1.0, 578: 1.0, 759: 1.0, 832: 2.0}), TFIDF=SparseVector(1000, {0: 0.0026, 1: 0.7515, 2: 0.1374, 3: 0.3184, 5: 0.3823, 8: 1.0754, 9: 0.3344, 15: 0.5899, 21: 1.8551, 28: 1.1263, 31: 1.1113, 35: 3.3134, 36: 1.2545, 43: 2.3741, 45: 2.3753, 48: 1.2254, 51: 1.1879, 57: 11.0264, 61: 2.8957, 71: 2.1945, 78: 1.6947, 84: 6.5898, 86: 1.6136, 94: 2.3569, 97: 1.8218, 99: 2.6292, 100: 1.9206, 115: 2.3592, 147: 5.4841, 152: 2.1116, 169: 2.6328, 241: 2.5745, 283: 3.2325, 306: 3.2668, 350: 6.2367, 490: 3.8893, 578: 3.6182, 759: 3.7771, 832: 8.8964}), label=3.0, pcaTFIDF=DenseVector([-0.5291, -0.8217, 0.3129, 0.02, 0.0323, -0.1371, 0.1032, -0.1511, 0.4816, -0.2657, 0.9031, -0.1125, 3.1339, -0.4338, -0.1165, 0.3052, 0.9695, -0.7508, 0.1898, -1.0876, 0.5371, -0.8804, 1.5681, -0.3721, -0.4511, 0.6415, 0.5597, -0.0773, 0.4399, 1.0323, -0.8446, 0.7257, -0.6349, -1.3363, -0.9206, 1.5778, -1.8451, -0.2224, -1.1524, -0.0381, -0.0415, 0.3505, 1.2341, -0.4662, 0.8383, 0.772, 0.7149, -1.0151, 0.148, 0.1278, -0.946, -0.6953, -1.5553, -0.9866, 0.7846, -0.7185, 0.946, 0.6609, -0.0182, 1.3281, -0.4261, -0.6093, -0.8237, -0.5232, -0.5305, -0.4872, 0.1315, 0.8463, -1.1532, -1.2489, 0.3981, -1.4053, 0.4366, -0.931, 0.062, 0.9369, 0.8366, -0.7272, 1.5533, -1.9902, -0.4451, 0.9578, 0.364, -0.3055, -0.9719, -1.1939, 1.1266, -0.3546, 1.6776, 2.1847, -0.0966, -1.6945, -0.9625, -0.7207, 0.4287, -0.6703, 0.7134, 0.2583, -1.692, -0.4525]), NumTags=5, LengthFeature=DenseVector([83.0]))"
      ]
     },
     "execution_count": 53,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# Returns the first ``n`` rows\n",
    "stack_overflow_data.head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 54,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Linear regression \n",
    "# The learning objective is to minimize the specified loss function, with regularization\n",
    "# This supports two kinds of loss:\n",
    "#   * squaredError (a.k.a squared loss)\n",
    "#   * huber (a hybrid of squared error for relatively small errors and absolute error for relatively large ones, and we estimate the scale parameter from training data)\n",
    "# This supports multiple types of regularization:\n",
    "#   * none (a.k.a. ordinary least squares)\n",
    "#   * L2 (ridge regression)\n",
    "#   * L1 (Lasso)\n",
    "#   * L2 + L1 (elastic net)\n",
    "# Note: Fitting with huber loss only supports none and L2 regularization\n",
    "lr = LinearRegression(maxIter=5, regParam=0.0, fitIntercept=False, solver='normal')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 55,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "Row(label=5, features=DenseVector([83.0]))"
      ]
     },
     "execution_count": 55,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "data = stack_overflow_data.select(col('NumTags').alias('label'), col('LengthFeature').alias('features'))\n",
    "\n",
    "# Returns the first ``n`` rows\n",
    "data.head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 56,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Fits a model to the input dataset with optional parameters\n",
    "lrModel = lr.fit(data)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 57,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "DenseVector([0.0079])"
      ]
     },
     "execution_count": 57,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# LinearRegression model coefficients\n",
    "lrModel.coefficients"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 58,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "0.0"
      ]
     },
     "execution_count": 58,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# LinearRegression model intercept\n",
    "lrModel.intercept"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 59,
   "metadata": {},
   "outputs": [],
   "source": [
    "# LinearRegression summary (e.g. residuals, mse, r-squared) of model on training set\n",
    "# An exception is thrown if `trainingSummary is None`\n",
    "lrModelSummary = lrModel.summary"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 60,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "0.42481762576079773"
      ]
     },
     "execution_count": 60,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# LinearRegression R^2, the coefficient of determination\n",
    "lrModelSummary.r2"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Question\n",
    "\n",
    "Build a linear regression model using the length of the combined `Title` + `Body` fields. What is the value of r^2 when fitting a model with `maxIter=5`, `regParam=0.0`, `fitIntercept=False`, `solver='normal'`?"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 61,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Returns a new `DataFrame` by adding a column or replacing the existing column that has the same name\n",
    "# concat: concatenates multiple input columns together into a single column;\n",
    "#         the function works with strings, binary and compatible array columns\n",
    "stack_overflow_data = stack_overflow_data.withColumn('Desc', concat(col('Title'), lit(' '), col('Body')))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 62,
   "metadata": {},
   "outputs": [],
   "source": [
    "# A regex based tokenizer that extracts tokens either by using the provided regex pattern (in Java dialect) to \n",
    "#    split the text (default) or repeatedly matching the regex (if gaps is false)\n",
    "# Optional parameters also allow filtering tokens using a minimal length\n",
    "# It returns an array of strings that can be empty \n",
    "regexTokenizer = RegexTokenizer(inputCol='Desc', outputCol='words2', pattern='\\\\W')\n",
    "\n",
    "# Transforms the input dataset with optional parameters\n",
    "stack_overflow_data = regexTokenizer.transform(stack_overflow_data)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 63,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Returns a new `DataFrame` by adding a column or replacing the existing column that has the same name\n",
    "stack_overflow_data = stack_overflow_data.withColumn('DescLength', body_length(stack_overflow_data.words2))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 64,
   "metadata": {},
   "outputs": [],
   "source": [
    "# A feature transformer that merges multiple columns into a vector column\n",
    "vecAssembler = VectorAssembler(inputCols=['DescLength'], outputCol='DescVec')\n",
    "\n",
    "# Transforms the input dataset with optional parameters\n",
    "stack_overflow_data = vecAssembler.transform(stack_overflow_data)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 65,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "+-------+------------------+\n",
      "|NumTags|   avg(DescLength)|\n",
      "+-------+------------------+\n",
      "|      1|143.68776158175783|\n",
      "|      2| 162.1539186134137|\n",
      "|      3|181.26021064340088|\n",
      "|      4|201.46530249110322|\n",
      "|      5|227.64375266524522|\n",
      "+-------+------------------+\n",
      "\n"
     ]
    }
   ],
   "source": [
    "stack_overflow_data.groupby('NumTags').agg(avg(col('DescLength'))).orderBy('NumTags').show()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 66,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Linear regression \n",
    "# The learning objective is to minimize the specified loss function, with regularization\n",
    "# This supports two kinds of loss:\n",
    "#   * squaredError (a.k.a squared loss)\n",
    "#   * huber (a hybrid of squared error for relatively small errors and absolute error for relatively large ones, and we estimate the scale parameter from training data)\n",
    "# This supports multiple types of regularization:\n",
    "#   * none (a.k.a. ordinary least squares)\n",
    "#   * L2 (ridge regression)\n",
    "#   * L1 (Lasso)\n",
    "#   * L2 + L1 (elastic net)\n",
    "# Note: Fitting with huber loss only supports none and L2 regularization\n",
    "lr = LinearRegression(maxIter=5, regParam=0.0, fitIntercept=False, solver='normal')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 67,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "[Row(label=5, features=DenseVector([96.0])),\n",
       " Row(label=1, features=DenseVector([83.0])),\n",
       " Row(label=3, features=DenseVector([3168.0])),\n",
       " Row(label=3, features=DenseVector([124.0])),\n",
       " Row(label=3, features=DenseVector([154.0]))]"
      ]
     },
     "execution_count": 67,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "data = stack_overflow_data.select(col('NumTags').alias('label'), col('DescVec').alias('features'))\n",
    "\n",
    "# Returns the first ``n`` rows\n",
    "data.head(5)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 68,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Fits a model to the input dataset with optional parameters\n",
    "lrModel = lr.fit(data)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 69,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "0.44551495963084176"
      ]
     },
     "execution_count": 69,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# LinearRegression R^2, the coefficient of determination\n",
    "lrModel.summary.r2"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Logistic Regression"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 70,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Logistic regression\n",
    "# This class supports multinomial logistic (softmax) and binomial logistic regression\n",
    "logreg = LogisticRegression(maxIter=10, regParam=0.0)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 71,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "Row(label=3.0, features=SparseVector(1000, {0: 0.0026, 1: 0.7515, 2: 0.1374, 3: 0.3184, 5: 0.3823, 8: 1.0754, 9: 0.3344, 15: 0.5899, 21: 1.8551, 28: 1.1263, 31: 1.1113, 35: 3.3134, 36: 1.2545, 43: 2.3741, 45: 2.3753, 48: 1.2254, 51: 1.1879, 57: 11.0264, 61: 2.8957, 71: 2.1945, 78: 1.6947, 84: 6.5898, 86: 1.6136, 94: 2.3569, 97: 1.8218, 99: 2.6292, 100: 1.9206, 115: 2.3592, 147: 5.4841, 152: 2.1116, 169: 2.6328, 241: 2.5745, 283: 3.2325, 306: 3.2668, 350: 6.2367, 490: 3.8893, 578: 3.6182, 759: 3.7771, 832: 8.8964}))"
      ]
     },
     "execution_count": 71,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# Note: Multinomial logistic regression\n",
    "data2 = stack_overflow_data.select(col('label').alias('label'), col('TFIDF').alias('features'))\n",
    "\n",
    "# Returns the first ``n`` rows\n",
    "data2.head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 72,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Fits a model to the input dataset with optional parameters\n",
    "logregModel = logreg.fit(data2)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 73,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "DenseMatrix(301, 1000, [7.2356, 0.0372, 0.0333, 0.0894, -0.0442, 0.0287, 0.0018, 0.0007, ..., -0.0006, -0.0009, -0.0002, -0.0003, -0.0, -0.0015, -0.0003, -0.0005], 1)"
      ]
     },
     "execution_count": 73,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# LogisticRegression model coefficients\n",
    "logregModel.coefficientMatrix"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 74,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "DenseVector([5.0624, 4.2809, 4.1836, 4.0456, 3.9815, 3.8424, 3.3918, 3.4562, 3.3316, 3.2418, 2.9428, 2.8218, 2.7839, 2.7625, 2.6392, 2.5983, 2.4539, 2.4447, 2.3916, 2.3566, 2.1003, 2.0631, 2.0567, 1.7878, 1.7815, 1.7789, 1.7183, 1.5344, 1.5141, 1.4106, 1.3633, 1.3618, 1.3407, 1.3321, 1.3387, 1.2438, 1.1902, 1.1985, 1.2037, 1.2022, 1.1798, 1.1327, 1.1006, 1.0406, 0.9521, 0.9417, 0.9192, 0.9164, 0.8901, 0.8584, 0.8452, 0.8359, 0.8296, 0.8064, 0.7944, 0.7899, 0.7819, 0.7776, 0.7598, 0.7628, 0.7327, 0.7291, 0.6964, 0.6557, 0.6597, 0.6572, 0.6451, 0.6439, 0.6062, 0.6087, 0.5191, 0.5071, 0.5063, 0.5012, 0.466, 0.4616, 0.4529, 0.4337, 0.4241, 0.4104, 0.406, 0.3852, 0.3536, 0.3461, 0.3453, 0.3236, 0.2877, 0.2839, 0.2742, 0.2597, 0.2447, 0.2194, 0.1946, 0.1855, 0.1849, 0.1718, 0.1684, 0.1625, 0.1454, 0.1231, 0.1156, 0.1063, 0.0996, 0.0669, 0.0713, 0.0213, 0.0143, 0.0013, 0.0025, -0.0089, -0.0199, -0.0291, -0.0283, -0.0409, -0.0398, -0.0501, -0.0721, -0.0735, -0.0711, -0.0729, -0.0839, -0.0852, -0.1064, -0.1079, -0.1196, -0.119, -0.1414, -0.141, -0.1536, -0.1663, -0.1767, -0.1784, -0.1899, -0.2027, -0.2145, -0.2156, -0.2283, -0.2274, -0.2282, -0.2416, -0.2537, -0.2544, -0.2683, -0.267, -0.2809, -0.2796, -0.2811, -0.2939, -0.307, -0.3094, -0.3221, -0.3217, -0.3507, -0.3491, -0.3655, -0.3647, -0.3653, -0.3815, -0.3792, -0.3962, -0.4106, -0.4101, -0.4106, -0.4254, -0.425, -0.4252, -0.4416, -0.4414, -0.443, -0.4416, -0.4428, -0.4567, -0.4586, -0.4738, -0.4749, -0.4907, -0.491, -0.4906, -0.491, -0.4903, -0.5075, -0.5088, -0.5072, -0.5244, -0.5436, -0.5421, -0.5602, -0.5768, -0.5786, -0.5774, -0.5776, -0.6143, -0.6148, -0.6145, -0.6148, -0.6339, -0.6339, -0.6533, -0.6534, -0.6732, -0.674, -0.673, -0.6936, -0.694, -0.7143, -0.7146, -0.7147, -0.7355, -0.7356, -0.7358, -0.735, -0.7351, -0.7572, -0.7577, -0.7565, -0.757, -0.7567, -0.7577, -0.7569, -0.7569, -0.7789, -0.7785, -0.8018, -0.8017, -0.8014, -0.8011, -0.8009, -0.8249, -0.8476, -0.8478, -0.8472, -0.848, -0.8717, -0.8729, -0.8721, -0.8969, -0.8975, -0.8966, -0.8963, -0.8976, -0.9218, -0.9223, -0.9221, -0.9223, -0.922, -0.9221, -0.922, -0.9481, -0.9478, -0.948, -0.9751, -0.9745, -0.9749, -0.9747, -0.9747, -0.9754, -1.0018, -1.0306, -1.0304, -1.0303, -1.0306, -1.0305, -1.0309, -1.0302, -1.0305, -1.0306, -1.0593, -1.0592, -1.059, -1.0591, -1.059, -1.0595, -1.0596, -1.0892, -1.0889, -1.089, -1.1204, -1.1201, -1.12, -1.1199, -1.1517, -1.1518, -1.1522, -1.1846, -1.1849, -1.185, -1.1846, -1.2187, -1.2191, -1.2187, -1.2538, -1.2904, -1.2904, -1.2902, -1.3671, -1.408, -1.4508, -1.4507, -1.4951, -1.5419, -1.5905])"
      ]
     },
     "execution_count": 74,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# LogisticRegression model intercept\n",
    "logregModel.interceptVector"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 75,
   "metadata": {},
   "outputs": [],
   "source": [
    "# LogisticRegression summary (e.g. accuracy/precision/recall, objective history, total iterations) of model on training set\n",
    "# An exception is thrown if `trainingSummary is None`\n",
    "logregModelSummary = logregModel.summary"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 76,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "0.3674"
      ]
     },
     "execution_count": 76,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# LogisticRegression accuracy (total number of correctly classified instances out of the total number of instances)\n",
    "logregModelSummary.accuracy"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 77,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "0.0033222591362126247"
      ]
     },
     "execution_count": 77,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# Random chance of getting the tag right\n",
    "1/301.0"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Unsupervised ML Algorithms"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### K-Means\n",
    "\n",
    "Examine the distribution of the Title + Body length feature used before and instead of using the raw number of words, create categories based on this length: short, longer, and super long."
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Question 1\n",
    "\n",
    "How many times greater is the Description Length of the longest question than the Description Length of the shortest question (rounded to the nearest whole number)?\n",
    "\n",
    "Tip: Don't forget to import Spark SQL's aggregate functions that can operate on DataFrame columns."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 78,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "+---------------+\n",
      "|min(DescLength)|\n",
      "+---------------+\n",
      "|             10|\n",
      "+---------------+\n",
      "\n"
     ]
    }
   ],
   "source": [
    "stack_overflow_data.agg(min('DescLength')).show()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 79,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "+---------------+\n",
      "|max(DescLength)|\n",
      "+---------------+\n",
      "|           7532|\n",
      "+---------------+\n",
      "\n"
     ]
    }
   ],
   "source": [
    "stack_overflow_data.agg(max('DescLength')).show()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 80,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "+-----------------------------------+\n",
      "|(max(DescLength) / min(DescLength))|\n",
      "+-----------------------------------+\n",
      "|                              753.2|\n",
      "+-----------------------------------+\n",
      "\n"
     ]
    }
   ],
   "source": [
    "stack_overflow_data.agg(max('DescLength')/min('DescLength')).show()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Question 2\n",
    "\n",
    "What is the mean and standard deviation of the Description length?"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 81,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "+---------------+-----------------------+\n",
      "|avg(DescLength)|stddev_samp(DescLength)|\n",
      "+---------------+-----------------------+\n",
      "|      180.28187|     192.10819533505128|\n",
      "+---------------+-----------------------+\n",
      "\n"
     ]
    }
   ],
   "source": [
    "stack_overflow_data.agg(avg('DescLength'), stddev('DescLength')).show()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Question 3\n",
    "\n",
    "Use K-means to create 5 clusters of Description lengths. Set the random seed to 42 and fit a 5-class K-means model on the Description length column (use `KMeans().setParams(...)`).\n",
    "\n",
    "What length is the center of the cluster representing the longest questions?"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 82,
   "metadata": {},
   "outputs": [],
   "source": [
    "# K-means clustering with a k-means++ like initialization mode (the k-means|| algorithm by Bahmani et al)\n",
    "# Sets params for KMeans\n",
    "kmeans = KMeans().setParams(featuresCol='DescVec', predictionCol='DescGroup', k=5, seed=42)\n",
    "\n",
    "# Fits a model to the input dataset with optional parameters\n",
    "kmeansModel = kmeans.fit(stack_overflow_data)\n",
    "\n",
    "# Transforms the input dataset with optional parameters\n",
    "stack_overflow_data = kmeansModel.transform(stack_overflow_data)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 83,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "Row(Body=\"<p>I'd like to check if an uploaded file is an image file (e.g png, jpg, jpeg, gif, bmp) or another file. The problem is that I'm using Uploadify to upload the files, which changes the mime type and gives a 'text/octal' or something as the mime type, no matter which file type you upload.</p>\\n\\n<p>Is there a way to check if the uploaded file is an image apart from checking the file extension using PHP?</p>\\n\", Id=1, Tags='php image-processing file-upload upload mime-types', Title='How to check if an uploaded file is an image without mime type?', oneTag='php', words=['p', 'i', 'd', 'like', 'to', 'check', 'if', 'an', 'uploaded', 'file', 'is', 'an', 'image', 'file', 'e', 'g', 'png', 'jpg', 'jpeg', 'gif', 'bmp', 'or', 'another', 'file', 'the', 'problem', 'is', 'that', 'i', 'm', 'using', 'uploadify', 'to', 'upload', 'the', 'files', 'which', 'changes', 'the', 'mime', 'type', 'and', 'gives', 'a', 'text', 'octal', 'or', 'something', 'as', 'the', 'mime', 'type', 'no', 'matter', 'which', 'file', 'type', 'you', 'upload', 'p', 'p', 'is', 'there', 'a', 'way', 'to', 'check', 'if', 'the', 'uploaded', 'file', 'is', 'an', 'image', 'apart', 'from', 'checking', 'the', 'file', 'extension', 'using', 'php', 'p'], BodyLength=83, NumParagraphs=2, NumLinks=0, NumFeatures=DenseVector([83.0, 2.0, 0.0]), ScaledNumFeatures=DenseVector([0.9997, 0.0241, 0.0]), ScaledNumFeatures2=DenseVector([0.4325, 0.7037, 0.0]), TF=SparseVector(1000, {0: 4.0, 1: 6.0, 2: 2.0, 3: 3.0, 5: 2.0, 8: 4.0, 9: 1.0, 15: 1.0, 21: 2.0, 28: 1.0, 31: 1.0, 35: 3.0, 36: 1.0, 43: 2.0, 45: 2.0, 48: 1.0, 51: 1.0, 57: 6.0, 61: 2.0, 71: 1.0, 78: 1.0, 84: 3.0, 86: 1.0, 94: 1.0, 97: 1.0, 99: 1.0, 100: 1.0, 115: 1.0, 147: 2.0, 152: 1.0, 169: 1.0, 241: 1.0, 283: 1.0, 306: 1.0, 350: 2.0, 490: 1.0, 578: 1.0, 759: 1.0, 832: 2.0}), TFIDF=SparseVector(1000, {0: 0.0026, 1: 0.7515, 2: 0.1374, 3: 0.3184, 5: 0.3823, 8: 1.0754, 9: 0.3344, 15: 0.5899, 21: 1.8551, 28: 1.1263, 31: 1.1113, 35: 3.3134, 36: 1.2545, 43: 2.3741, 45: 2.3753, 48: 1.2254, 51: 1.1879, 57: 11.0264, 61: 2.8957, 71: 2.1945, 78: 1.6947, 84: 6.5898, 86: 1.6136, 94: 2.3569, 97: 1.8218, 99: 2.6292, 100: 1.9206, 115: 2.3592, 147: 5.4841, 152: 2.1116, 169: 2.6328, 241: 2.5745, 283: 3.2325, 306: 3.2668, 350: 6.2367, 490: 3.8893, 578: 3.6182, 759: 3.7771, 832: 8.8964}), label=3.0, pcaTFIDF=DenseVector([-0.5291, -0.8217, 0.3129, 0.02, 0.0323, -0.1371, 0.1032, -0.1511, 0.4816, -0.2657, 0.9031, -0.1125, 3.1339, -0.4338, -0.1165, 0.3052, 0.9695, -0.7508, 0.1898, -1.0876, 0.5371, -0.8804, 1.5681, -0.3721, -0.4511, 0.6415, 0.5597, -0.0773, 0.4399, 1.0323, -0.8446, 0.7257, -0.6349, -1.3363, -0.9206, 1.5778, -1.8451, -0.2224, -1.1524, -0.0381, -0.0415, 0.3505, 1.2341, -0.4662, 0.8383, 0.772, 0.7149, -1.0151, 0.148, 0.1278, -0.946, -0.6953, -1.5553, -0.9866, 0.7846, -0.7185, 0.946, 0.6609, -0.0182, 1.3281, -0.4261, -0.6093, -0.8237, -0.5232, -0.5305, -0.4872, 0.1315, 0.8463, -1.1532, -1.2489, 0.3981, -1.4053, 0.4366, -0.931, 0.062, 0.9369, 0.8366, -0.7272, 1.5533, -1.9902, -0.4451, 0.9578, 0.364, -0.3055, -0.9719, -1.1939, 1.1266, -0.3546, 1.6776, 2.1847, -0.0966, -1.6945, -0.9625, -0.7207, 0.4287, -0.6703, 0.7134, 0.2583, -1.692, -0.4525]), NumTags=5, LengthFeature=DenseVector([83.0]), Desc=\"How to check if an uploaded file is an image without mime type? <p>I'd like to check if an uploaded file is an image file (e.g png, jpg, jpeg, gif, bmp) or another file. The problem is that I'm using Uploadify to upload the files, which changes the mime type and gives a 'text/octal' or something as the mime type, no matter which file type you upload.</p>\\n\\n<p>Is there a way to check if the uploaded file is an image apart from checking the file extension using PHP?</p>\\n\", words2=['how', 'to', 'check', 'if', 'an', 'uploaded', 'file', 'is', 'an', 'image', 'without', 'mime', 'type', 'p', 'i', 'd', 'like', 'to', 'check', 'if', 'an', 'uploaded', 'file', 'is', 'an', 'image', 'file', 'e', 'g', 'png', 'jpg', 'jpeg', 'gif', 'bmp', 'or', 'another', 'file', 'the', 'problem', 'is', 'that', 'i', 'm', 'using', 'uploadify', 'to', 'upload', 'the', 'files', 'which', 'changes', 'the', 'mime', 'type', 'and', 'gives', 'a', 'text', 'octal', 'or', 'something', 'as', 'the', 'mime', 'type', 'no', 'matter', 'which', 'file', 'type', 'you', 'upload', 'p', 'p', 'is', 'there', 'a', 'way', 'to', 'check', 'if', 'the', 'uploaded', 'file', 'is', 'an', 'image', 'apart', 'from', 'checking', 'the', 'file', 'extension', 'using', 'php', 'p'], DescLength=96, DescVec=DenseVector([96.0]), DescGroup=4)"
      ]
     },
     "execution_count": 83,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# Returns the first ``n`` rows\n",
    "stack_overflow_data.head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 84,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "+---------+------------------+------------------+-----------------+\n",
      "|DescGroup|   avg(DescLength)|      avg(NumTags)|count(DescLength)|\n",
      "+---------+------------------+------------------+-----------------+\n",
      "|        4| 92.75317245164402| 2.732166913366707|            60127|\n",
      "|        0|224.90495069296375| 3.068663379530917|            30016|\n",
      "|        2| 457.1547183613753|3.2275054864667156|             8202|\n",
      "|        3| 989.9467576791809| 3.279180887372014|             1465|\n",
      "|        1| 2634.815789473684|3.3684210526315788|              190|\n",
      "+---------+------------------+------------------+-----------------+\n",
      "\n"
     ]
    }
   ],
   "source": [
    "stack_overflow_data.groupby('DescGroup').agg(avg(col('DescLength')), avg(col('NumTags')), count(col('DescLength'))).orderBy('avg(DescLength)').show()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# ML Pipelines"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 85,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "DataFrame[Body: string, Id: bigint, Tags: string, Title: string, oneTag: string]"
      ]
     },
     "execution_count": 85,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "path = '/Users/yangweichle/Documents/Employment/TRAINING/DATA SCIENCE/Spark/Udacity_Spark for Big Data/Machine Learning with Spark/data/Train_onetag_small.json'\n",
    "\n",
    "# Loads JSON files and returns the results as a `DataFrame`\n",
    "# Note: path: string represents path to the JSON dataset, or a list of paths, or RDD of Strings storing JSON objects\n",
    "stack_overflow_data = spark.read.json(path=path)\n",
    "\n",
    "# Sets the storage level to persist the contents of the `DataFrame` across operations after the first time it is computed\n",
    "# This can only be used to assign a new storage level if the `DataFrame` does not have a storage level set yet\n",
    "# If no storage level is specified defaults to (C{MEMORY_AND_DISK})\n",
    "stack_overflow_data.persist()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 86,
   "metadata": {},
   "outputs": [],
   "source": [
    "# A regex based tokenizer that extracts tokens either by using the provided regex pattern (in Java dialect) to \n",
    "#    split the text (default) or repeatedly matching the regex (if gaps is false)\n",
    "# Optional parameters also allow filtering tokens using a minimal length\n",
    "# It returns an array of strings that can be empty\n",
    "regexTokenizer = RegexTokenizer(inputCol='Body', outputCol='words', pattern='\\\\W')\n",
    "\n",
    "# Extracts a vocabulary from document collections and generates a `CountVectorizerModel`\n",
    "cv = CountVectorizer(inputCol='words', outputCol='TF', vocabSize=1000)\n",
    "\n",
    "# Compute the Inverse Document Frequency (IDF) given a collection of documents\n",
    "idf = IDF(inputCol='TF', outputCol='features')\n",
    "\n",
    "# A label indexer that maps a string column of labels to an ML column of label indices\n",
    "# If the input column is numeric, we cast it to string and index the string values\n",
    "# The indices are in [0, numLabels). By default, this is ordered by label frequencies so the most frequent label gets index 0\n",
    "# Note: stringOrderType: the ordering behavior; default value is 'frequencyDesc'; other option is 'alphabetDesc'\n",
    "stringIndexer = StringIndexer(inputCol='oneTag', outputCol='label')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 87,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Logistic regression\n",
    "# This class supports multinomial logistic (softmax) and binomial logistic regression\n",
    "logreg = LogisticRegression(maxIter=10, regParam=0.0, elasticNetParam=0)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 88,
   "metadata": {},
   "outputs": [],
   "source": [
    "# A simple pipeline, which acts as an estimator\n",
    "# A Pipeline consists of a sequence of stages, each of which is either an `Estimator` or a `Transformer`\n",
    "# When `Pipeline.fit` is called, the stages are executed in order\n",
    "# If a stage is an `Estimator`, its `Estimator.fit` method will be called on the input dataset to fit a model\n",
    "#   Then the model, which is a transformer, will be used to transform the dataset as the input to the next stage\n",
    "# If a stage is a `Transformer`, its `Transformer.transform` method will be called to produce the dataset for the next stage\n",
    "# The fitted model from a `Pipeline` is a `PipelineModel`, which consists of fitted models and transformers, corresponding to the pipeline stages\n",
    "# If stages is an empty list, the pipeline acts as an identity transformer\n",
    "pipeline = Pipeline(stages=[regexTokenizer, cv, idf, stringIndexer, logreg])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 89,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Fits a model to the input dataset with optional parameters\n",
    "plogregModel = pipeline.fit(stack_overflow_data)\n",
    "\n",
    "# Transforms the input dataset with optional parameters\n",
    "stack_overflow_data = plogregModel.transform(stack_overflow_data)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 90,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "Row(Body=\"<p>I'd like to check if an uploaded file is an image file (e.g png, jpg, jpeg, gif, bmp) or another file. The problem is that I'm using Uploadify to upload the files, which changes the mime type and gives a 'text/octal' or something as the mime type, no matter which file type you upload.</p>\\n\\n<p>Is there a way to check if the uploaded file is an image apart from checking the file extension using PHP?</p>\\n\", Id=1, Tags='php image-processing file-upload upload mime-types', Title='How to check if an uploaded file is an image without mime type?', oneTag='php', words=['p', 'i', 'd', 'like', 'to', 'check', 'if', 'an', 'uploaded', 'file', 'is', 'an', 'image', 'file', 'e', 'g', 'png', 'jpg', 'jpeg', 'gif', 'bmp', 'or', 'another', 'file', 'the', 'problem', 'is', 'that', 'i', 'm', 'using', 'uploadify', 'to', 'upload', 'the', 'files', 'which', 'changes', 'the', 'mime', 'type', 'and', 'gives', 'a', 'text', 'octal', 'or', 'something', 'as', 'the', 'mime', 'type', 'no', 'matter', 'which', 'file', 'type', 'you', 'upload', 'p', 'p', 'is', 'there', 'a', 'way', 'to', 'check', 'if', 'the', 'uploaded', 'file', 'is', 'an', 'image', 'apart', 'from', 'checking', 'the', 'file', 'extension', 'using', 'php', 'p'], TF=SparseVector(1000, {0: 4.0, 1: 6.0, 2: 2.0, 3: 3.0, 5: 2.0, 8: 4.0, 9: 1.0, 15: 1.0, 21: 2.0, 28: 1.0, 31: 1.0, 35: 3.0, 36: 1.0, 43: 2.0, 45: 2.0, 48: 1.0, 51: 1.0, 57: 6.0, 61: 2.0, 71: 1.0, 78: 1.0, 84: 3.0, 86: 1.0, 94: 1.0, 97: 1.0, 99: 1.0, 100: 1.0, 115: 1.0, 147: 2.0, 152: 1.0, 169: 1.0, 241: 1.0, 283: 1.0, 306: 1.0, 350: 2.0, 490: 1.0, 578: 1.0, 759: 1.0, 832: 2.0}), features=SparseVector(1000, {0: 0.0026, 1: 0.7515, 2: 0.1374, 3: 0.3184, 5: 0.3823, 8: 1.0754, 9: 0.3344, 15: 0.5899, 21: 1.8551, 28: 1.1263, 31: 1.1113, 35: 3.3134, 36: 1.2545, 43: 2.3741, 45: 2.3753, 48: 1.2254, 51: 1.1879, 57: 11.0264, 61: 2.8957, 71: 2.1945, 78: 1.6947, 84: 6.5898, 86: 1.6136, 94: 2.3569, 97: 1.8218, 99: 2.6292, 100: 1.9206, 115: 2.3592, 147: 5.4841, 152: 2.1116, 169: 2.6328, 241: 2.5745, 283: 3.2325, 306: 3.2668, 350: 6.2367, 490: 3.8893, 578: 3.6182, 759: 3.7771, 832: 8.8964}), label=3.0, rawPrediction=DenseVector([4.3561, 3.7158, 3.9252, 6.5266, 2.6863, 3.376, 2.1501, 3.0103, 3.0963, 0.947, 2.7247, 1.351, 2.6692, 1.0837, 0.9419, 2.3994, 1.3008, 2.114, 1.9185, 2.1695, 1.9607, 0.9716, 0.886, 1.4527, 0.9932, 1.9327, 1.6632, 1.3998, 1.2302, 1.1797, 1.3151, 1.0156, 1.2078, 0.7854, 0.9102, 1.3521, 0.6727, 1.4952, 1.8241, 1.1594, 0.946, 1.1233, 1.156, 1.239, 0.8427, 0.7222, 0.8731, 1.0679, 1.1605, 1.1331, 0.7814, 0.9909, 0.9034, 1.2087, 0.8857, 1.4443, 0.9715, 0.4755, 0.5806, 0.3858, 0.7771, 0.9584, 0.8699, 0.7622, 0.5561, 0.7465, 0.8109, 0.8665, 0.4109, 0.846, 1.058, 0.4833, 0.6517, 0.5909, 0.6856, 0.4652, 0.4635, 0.6983, 0.4035, 0.4452, 0.5067, 0.3834, 0.2736, 0.3747, 0.5158, 0.4308, 0.5393, 0.5154, 0.5042, 0.4782, 0.3438, 0.1493, 0.3453, 0.3128, 0.3071, 0.1383, 0.1796, 0.3869, 0.141, 0.2517, 0.2157, 0.2808, 0.2961, 0.4207, 0.0604, 0.0207, -0.1891, 0.5276, -0.0765, -0.0512, 0.0686, 0.1779, -0.1, 0.0045, -0.0632, -0.091, -0.0577, 0.2276, -0.126, -0.1439, -0.0548, 0.0039, -0.0412, 0.008, 0.033, 0.1884, -0.1182, 0.2282, -0.1017, -0.0642, -0.2003, -0.0751, -0.159, 0.1302, -0.2261, 0.1558, -0.0425, -0.1892, -0.1318, -0.1542, -0.2054, -0.1944, 0.0133, -0.3427, -0.2235, -0.1651, -0.2, -0.1063, -0.3925, -0.2345, -0.2734, -0.1548, -0.2451, -0.4066, -0.2937, -0.3587, -0.3354, -0.2733, -0.308, -0.2021, -0.3738, -0.4513, -0.3034, -0.4439, -0.3727, -0.4228, -0.4232, -0.3714, -0.3049, -0.4013, -0.4104, -0.5534, -0.35, -0.4141, -0.4, -0.4162, -0.3763, -0.3242, -0.4028, -0.4575, -0.4279, -0.0313, -0.5772, -0.4781, -0.3972, -0.4862, -0.5334, -0.6531, -0.2726, -0.6126, -0.5029, -0.5182, -0.6167, -0.6075, -0.617, -0.6298, -0.6246, -0.5706, -0.5923, -0.6146, -0.4167, -0.7126, -0.6724, -0.5153, -0.6442, -0.5438, -0.5776, -0.6957, -0.5837, -0.6565, -0.75, -0.7571, -0.6838, -0.6246, -0.7499, -0.6193, -0.6875, -0.6051, -0.6539, -0.7522, -0.7032, -0.7013, -0.7024, -0.7582, -0.616, -0.8041, -0.8194, -0.7518, -0.7783, -0.796, -0.9197, -0.8047, -0.8978, -0.7639, -0.8221, -0.7903, -0.8267, -0.8331, -0.9169, -0.7712, -0.9191, -0.7988, -0.8821, -0.8377, -0.9422, -0.815, -0.9167, -0.8556, -0.9306, -0.9078, -0.895, -0.95, -0.9352, -0.9628, -0.9648, -0.8278, -1.0541, -0.9911, -1.0197, -1.0225, -0.9933, -0.9506, -0.8358, -0.8798, -1.0145, -0.9673, -1.0825, -1.0587, -1.1202, -1.1145, -1.1208, -0.9899, -1.0284, -1.0947, -1.1578, -1.1414, -1.0474, -1.1412, -1.0983, -1.1428, -1.1113, -1.1368, -1.0908, -1.1877, -1.0879, -1.1321, -1.1498, -1.1429, -1.192, -1.2172, -1.2251, -1.2287, -1.2379, -1.2352, -1.3865, -1.4595, -1.386, -1.449, -1.5266, -1.4852, -1.5841]), probability=DenseVector([0.0578, 0.0305, 0.0376, 0.5063, 0.0109, 0.0217, 0.0064, 0.015, 0.0164, 0.0019, 0.0113, 0.0029, 0.0107, 0.0022, 0.0019, 0.0082, 0.0027, 0.0061, 0.005, 0.0065, 0.0053, 0.002, 0.0018, 0.0032, 0.002, 0.0051, 0.0039, 0.003, 0.0025, 0.0024, 0.0028, 0.002, 0.0025, 0.0016, 0.0018, 0.0029, 0.0015, 0.0033, 0.0046, 0.0024, 0.0019, 0.0023, 0.0024, 0.0026, 0.0017, 0.0015, 0.0018, 0.0022, 0.0024, 0.0023, 0.0016, 0.002, 0.0018, 0.0025, 0.0018, 0.0031, 0.002, 0.0012, 0.0013, 0.0011, 0.0016, 0.0019, 0.0018, 0.0016, 0.0013, 0.0016, 0.0017, 0.0018, 0.0011, 0.0017, 0.0021, 0.0012, 0.0014, 0.0013, 0.0015, 0.0012, 0.0012, 0.0015, 0.0011, 0.0012, 0.0012, 0.0011, 0.001, 0.0011, 0.0012, 0.0011, 0.0013, 0.0012, 0.0012, 0.0012, 0.001, 0.0009, 0.001, 0.001, 0.001, 0.0009, 0.0009, 0.0011, 0.0009, 0.001, 0.0009, 0.001, 0.001, 0.0011, 0.0008, 0.0008, 0.0006, 0.0013, 0.0007, 0.0007, 0.0008, 0.0009, 0.0007, 0.0007, 0.0007, 0.0007, 0.0007, 0.0009, 0.0007, 0.0006, 0.0007, 0.0007, 0.0007, 0.0007, 0.0008, 0.0009, 0.0007, 0.0009, 0.0007, 0.0007, 0.0006, 0.0007, 0.0006, 0.0008, 0.0006, 0.0009, 0.0007, 0.0006, 0.0006, 0.0006, 0.0006, 0.0006, 0.0008, 0.0005, 0.0006, 0.0006, 0.0006, 0.0007, 0.0005, 0.0006, 0.0006, 0.0006, 0.0006, 0.0005, 0.0006, 0.0005, 0.0005, 0.0006, 0.0005, 0.0006, 0.0005, 0.0005, 0.0005, 0.0005, 0.0005, 0.0005, 0.0005, 0.0005, 0.0005, 0.0005, 0.0005, 0.0004, 0.0005, 0.0005, 0.0005, 0.0005, 0.0005, 0.0005, 0.0005, 0.0005, 0.0005, 0.0007, 0.0004, 0.0005, 0.0005, 0.0005, 0.0004, 0.0004, 0.0006, 0.0004, 0.0004, 0.0004, 0.0004, 0.0004, 0.0004, 0.0004, 0.0004, 0.0004, 0.0004, 0.0004, 0.0005, 0.0004, 0.0004, 0.0004, 0.0004, 0.0004, 0.0004, 0.0004, 0.0004, 0.0004, 0.0004, 0.0003, 0.0004, 0.0004, 0.0004, 0.0004, 0.0004, 0.0004, 0.0004, 0.0003, 0.0004, 0.0004, 0.0004, 0.0003, 0.0004, 0.0003, 0.0003, 0.0003, 0.0003, 0.0003, 0.0003, 0.0003, 0.0003, 0.0003, 0.0003, 0.0003, 0.0003, 0.0003, 0.0003, 0.0003, 0.0003, 0.0003, 0.0003, 0.0003, 0.0003, 0.0003, 0.0003, 0.0003, 0.0003, 0.0003, 0.0003, 0.0003, 0.0003, 0.0003, 0.0003, 0.0003, 0.0003, 0.0003, 0.0003, 0.0003, 0.0003, 0.0003, 0.0003, 0.0003, 0.0003, 0.0003, 0.0003, 0.0003, 0.0002, 0.0002, 0.0002, 0.0003, 0.0003, 0.0002, 0.0002, 0.0002, 0.0003, 0.0002, 0.0002, 0.0002, 0.0002, 0.0002, 0.0002, 0.0002, 0.0002, 0.0002, 0.0002, 0.0002, 0.0002, 0.0002, 0.0002, 0.0002, 0.0002, 0.0002, 0.0002, 0.0002, 0.0002, 0.0002, 0.0002, 0.0002, 0.0002]), prediction=3.0)"
      ]
     },
     "execution_count": 90,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# Returns the first ``n`` rows\n",
    "stack_overflow_data.head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 91,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "36740"
      ]
     },
     "execution_count": 91,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# Total number of correctly classified instances\n",
    "stack_overflow_data.where(stack_overflow_data.label == stack_overflow_data.prediction).count()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 92,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "100000"
      ]
     },
     "execution_count": 92,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# Total number of instances\n",
    "stack_overflow_data.count()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 93,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "0.3674"
      ]
     },
     "execution_count": 93,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# Accuracy = total number of correctly classified instances / total number of instances\n",
    "(stack_overflow_data.where(stack_overflow_data.label == stack_overflow_data.prediction).count())/stack_overflow_data.count()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 94,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Evaluator for Multiclass Classification, which expects two input columns: prediction and label\n",
    "evaluator = MulticlassClassificationEvaluator(predictionCol='prediction', labelCol='label', metricName='accuracy')\n",
    "\n",
    "# Evaluates the output with optional parameters\n",
    "accuracy = evaluator.evaluate(stack_overflow_data)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 95,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "0.3674"
      ]
     },
     "execution_count": 95,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# LogisticRegression accuracy (total number of correctly classified instances out of the total number of instances)\n",
    "accuracy"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Model Selection and Tuning"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 96,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "DataFrame[Body: string, Id: bigint, Tags: string, Title: string, oneTag: string]"
      ]
     },
     "execution_count": 96,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "path = '/Users/yangweichle/Documents/Employment/TRAINING/DATA SCIENCE/Spark/Udacity_Spark for Big Data/Machine Learning with Spark/data/Train_onetag_small.json'\n",
    "\n",
    "# Loads JSON files and returns the results as a `DataFrame`\n",
    "# Note: path: string represents path to the JSON dataset, or a list of paths, or RDD of Strings storing JSON objects\n",
    "stack_overflow_data = spark.read.json(path=path)\n",
    "\n",
    "# Sets the storage level to persist the contents of the `DataFrame` across operations after the first time it is computed\n",
    "# This can only be used to assign a new storage level if the `DataFrame` does not have a storage level set yet\n",
    "# If no storage level is specified defaults to (C{MEMORY_AND_DISK})\n",
    "stack_overflow_data.persist()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Step 1. Train Test Split\n",
    "\n",
    "As a first step break your data set into 80% of training data and set aside 20%. Set random seed to `42`."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 97,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Randomly splits this `DataFrame` with the provided weights\n",
    "train, test = stack_overflow_data.randomSplit([0.8, 0.2], seed=42)\n",
    "\n",
    "# Train, Test, Validation sets\n",
    "#train, rest = stack_overflow_data.randomSplit([0.6, 0.4], seed=42)\n",
    "#test, validation = rest.randomSplit([0.5, 0.5], seed=42)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Step 2. Build Pipeline"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 98,
   "metadata": {},
   "outputs": [],
   "source": [
    "# A regex based tokenizer that extracts tokens either by using the provided regex pattern (in Java dialect) to \n",
    "#    split the text (default) or repeatedly matching the regex (if gaps is false)\n",
    "# Optional parameters also allow filtering tokens using a minimal length\n",
    "# It returns an array of strings that can be empty\n",
    "regexTokenizer = RegexTokenizer(inputCol='Body', outputCol='words', pattern='\\\\W')\n",
    "\n",
    "# Extracts a vocabulary from document collections and generates a `CountVectorizerModel`\n",
    "cv = CountVectorizer(inputCol='words', outputCol='TF', vocabSize=1000)\n",
    "\n",
    "# Compute the Inverse Document Frequency (IDF) given a collection of documents\n",
    "idf = IDF(inputCol='TF', outputCol='features')\n",
    "\n",
    "# A label indexer that maps a string column of labels to an ML column of label indices\n",
    "# If the input column is numeric, we cast it to string and index the string values\n",
    "# The indices are in [0, numLabels). By default, this is ordered by label frequencies so the most frequent label gets index 0\n",
    "# Note: stringOrderType: the ordering behavior; default value is 'frequencyDesc'; other option is 'alphabetDesc'\n",
    "stringIndexer = StringIndexer(inputCol='oneTag', outputCol='label')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 99,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Logistic regression\n",
    "# This class supports multinomial logistic (softmax) and binomial logistic regression\n",
    "logreg = LogisticRegression(maxIter=10, regParam=0.0, elasticNetParam=0)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 100,
   "metadata": {},
   "outputs": [],
   "source": [
    "# A simple pipeline, which acts as an estimator\n",
    "# A Pipeline consists of a sequence of stages, each of which is either an `Estimator` or a `Transformer`\n",
    "# When `Pipeline.fit` is called, the stages are executed in order\n",
    "# If a stage is an `Estimator`, its `Estimator.fit` method will be called on the input dataset to fit a model\n",
    "#   Then the model, which is a transformer, will be used to transform the dataset as the input to the next stage\n",
    "# If a stage is a `Transformer`, its `Transformer.transform` method will be called to produce the dataset for the next stage\n",
    "# The fitted model from a `Pipeline` is a `PipelineModel`, which consists of fitted models and transformers, corresponding to the pipeline stages\n",
    "# If stages is an empty list, the pipeline acts as an identity transformer\n",
    "pipeline = Pipeline(stages=[regexTokenizer, cv, idf, stringIndexer, logreg])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 101,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Fits a model to the input dataset with optional parameters\n",
    "plogregModel = pipeline.fit(train)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 102,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Transforms the input dataset with optional parameters\n",
    "results = plogregModel.transform(test)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 103,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "Row(Body='<blockquote>\\n  <p><strong>Possible Duplicate:</strong><br>\\n  <a href=\"http://cstheory.stackexchange.com/questions/1574/do-you-use-any-article-organizers\">Do you use any article organizers?</a>  </p>\\n</blockquote>\\n\\n\\n\\n<p>As part of my Ph.D. studies I need to create an overview of recently (since 2000) published papers with impact on my study field.</p>\\n\\n<p>Creating a list of articles isn\\'t a problem, but what I also want to do is to follow the most cited articles and authors (after I\\'m done with this).</p>\\n\\n<p>Is there some tool, that would allow me to store the articles and then simplify the most cited authors/articles search?</p>\\n', Id=6835, Tags='soft-question software citations', Title='Tool for managing scientific papers', oneTag='soft-question', words=['blockquote', 'p', 'strong', 'possible', 'duplicate', 'strong', 'br', 'a', 'href', 'http', 'cstheory', 'stackexchange', 'com', 'questions', '1574', 'do', 'you', 'use', 'any', 'article', 'organizers', 'do', 'you', 'use', 'any', 'article', 'organizers', 'a', 'p', 'blockquote', 'p', 'as', 'part', 'of', 'my', 'ph', 'd', 'studies', 'i', 'need', 'to', 'create', 'an', 'overview', 'of', 'recently', 'since', '2000', 'published', 'papers', 'with', 'impact', 'on', 'my', 'study', 'field', 'p', 'p', 'creating', 'a', 'list', 'of', 'articles', 'isn', 't', 'a', 'problem', 'but', 'what', 'i', 'also', 'want', 'to', 'do', 'is', 'to', 'follow', 'the', 'most', 'cited', 'articles', 'and', 'authors', 'after', 'i', 'm', 'done', 'with', 'this', 'p', 'p', 'is', 'there', 'some', 'tool', 'that', 'would', 'allow', 'me', 'to', 'store', 'the', 'articles', 'and', 'then', 'simplify', 'the', 'most', 'cited', 'authors', 'articles', 'search', 'p'], TF=SparseVector(1000, {0: 8.0, 1: 3.0, 2: 3.0, 3: 4.0, 5: 4.0, 8: 2.0, 9: 2.0, 12: 1.0, 13: 3.0, 15: 1.0, 20: 2.0, 22: 1.0, 23: 1.0, 24: 2.0, 28: 1.0, 29: 1.0, 34: 1.0, 35: 1.0, 36: 1.0, 37: 2.0, 39: 3.0, 41: 1.0, 51: 1.0, 55: 1.0, 56: 2.0, 62: 1.0, 63: 1.0, 64: 2.0, 69: 1.0, 77: 1.0, 79: 2.0, 83: 1.0, 90: 1.0, 91: 1.0, 97: 1.0, 113: 1.0, 114: 1.0, 131: 1.0, 149: 1.0, 150: 1.0, 179: 2.0, 190: 1.0, 194: 1.0, 292: 1.0, 320: 1.0, 350: 1.0, 351: 1.0, 365: 1.0, 393: 1.0, 470: 1.0, 515: 2.0, 555: 1.0, 635: 1.0, 662: 1.0, 948: 1.0, 955: 1.0, 961: 2.0}), features=SparseVector(1000, {0: 0.005, 1: 0.3746, 2: 0.2055, 3: 0.4239, 5: 0.7666, 8: 0.5382, 9: 0.6692, 12: 0.4689, 13: 1.537, 15: 0.5916, 20: 1.6361, 22: 0.8387, 23: 0.6738, 24: 1.6268, 28: 1.127, 29: 1.1136, 34: 1.4936, 35: 1.2549, 36: 1.1061, 37: 4.1513, 39: 3.3401, 41: 1.6236, 51: 1.1838, 55: 1.2119, 56: 2.3928, 62: 1.392, 63: 1.4919, 64: 2.9257, 69: 1.5026, 77: 1.7258, 79: 3.3933, 83: 1.5816, 90: 1.7058, 91: 1.7969, 97: 1.8216, 113: 2.7588, 114: 2.3514, 131: 2.5528, 149: 2.235, 150: 2.0668, 179: 5.9697, 190: 2.3833, 194: 2.3009, 292: 2.8335, 320: 3.3304, 350: 3.5068, 351: 2.9782, 365: 3.0239, 393: 3.1972, 470: 3.5612, 515: 6.663, 555: 3.4498, 635: 3.6932, 662: 3.597, 948: 4.066, 955: 4.2008, 961: 8.9292}), label=291.0, rawPrediction=DenseVector([6.0352, 2.9448, 3.0642, 2.3046, 2.6565, 1.8008, 2.7443, 2.3928, 3.0844, -0.8741, 2.1228, 1.6621, 0.9153, 1.3625, 0.7972, 1.9495, 0.6201, 2.5194, 0.9281, 2.2586, 1.5576, 1.1972, 0.7323, 0.9838, 1.8504, 0.3972, 1.9386, 0.9947, 1.1279, 0.9168, 0.8247, 0.8259, 1.02, 0.8948, 1.166, 0.9897, 0.4112, 1.4569, 1.5847, 1.43, 0.8299, 1.5658, 1.2275, 1.3063, 1.3712, 1.2331, 1.5254, 1.3455, 1.0328, 0.9679, 1.0717, 0.8847, 0.6934, 0.8478, 0.7068, 0.5204, 0.5943, 0.7727, 0.9374, 0.8657, 1.2597, 0.9908, 1.0667, 1.059, 0.5745, 0.818, 0.4573, 0.4369, 0.8745, 0.6339, 0.9269, 0.5195, 0.6427, 0.4495, 0.748, 0.7949, 0.4853, 0.2278, 0.806, 0.505, 0.3799, 0.6979, 0.6028, 0.6318, 0.5096, 0.693, 0.5808, 0.5112, 0.5013, 0.4525, 0.7063, 0.2537, 0.3974, 0.3728, 0.3819, 0.1449, 0.251, 0.096, 0.3846, 0.2552, -0.2286, 0.3754, 0.363, 0.1957, 0.2627, -0.2738, 0.0737, 0.2541, 0.1846, 0.0079, 0.0228, 0.3192, 0.0847, 0.072, 0.3881, 0.0771, 0.1922, -0.1423, 0.0765, -0.236, -0.138, -0.1693, 0.0338, -0.1333, 0.0155, 0.0524, 0.0848, -0.149, 0.1543, 0.1496, 0.0151, 0.0489, -0.2501, 0.0077, -0.0268, -0.0409, 0.1454, -0.1115, 0.0145, -0.1212, -0.208, 0.0206, 0.0847, -0.2729, -0.135, -0.0627, 0.2058, -0.307, -0.0507, -0.1877, -0.4927, -0.3628, -0.0368, -0.1129, -0.3851, -0.2164, -0.3011, -0.1653, -0.1341, -0.0626, -0.3567, -0.3764, -0.2497, -0.4447, -0.1716, -0.4005, -0.432, -0.3126, -0.34, -0.2229, -0.3387, -0.0646, 0.0556, -0.3415, -0.3038, -0.3294, 0.0721, -0.4427, -0.3599, -0.5015, -0.2258, -0.4101, -0.3944, -0.5097, -0.3642, -0.4017, -0.3861, -0.4634, -0.546, -0.6411, -0.5076, -0.4293, -0.5972, -0.2957, -0.5455, -0.2965, -0.6031, -0.8614, -0.4778, -0.6701, -0.521, -0.3903, -0.5055, -0.423, -0.5156, -0.7208, -0.6778, -0.4105, -0.5779, -0.5905, -0.7221, -0.6211, -0.7882, -0.5087, -0.7542, -0.4929, -0.603, -0.8309, -0.9678, -0.5869, -0.329, -0.7966, -0.5988, -0.6291, -0.6814, -0.7765, -0.613, -0.3136, -0.6113, -0.5356, -0.7354, -0.7095, -0.5337, -0.7004, -0.7874, -0.7801, -0.6934, -0.9286, -0.7673, -0.9833, -0.8621, -0.7843, -0.9156, -0.8697, -0.914, -1.0146, -0.5111, -0.9619, -0.8726, -0.9176, -0.6892, -0.9549, -0.9034, -0.8209, -1.0441, -0.9596, -0.9849, -0.7339, -0.8824, -0.9836, -0.9881, -1.0012, -0.9591, -0.921, -0.8361, -0.9021, -0.8475, -1.0498, -0.9688, -0.9817, -0.7877, -1.0903, -0.8529, -1.0457, -1.2387, -1.104, -0.9226, -1.2294, -0.8831, -1.0774, -1.1809, -1.2304, -1.0504, -1.2282, -1.0914, -1.1878, -1.0981, -1.3166, -1.2317, -1.132, -1.2599, -1.0473, -1.299, -1.1538, -1.4437, -1.4511, -1.4167, -1.4956, -1.5714, -1.49, -1.5942]), probability=DenseVector([0.4536, 0.0206, 0.0232, 0.0109, 0.0155, 0.0066, 0.0169, 0.0119, 0.0237, 0.0005, 0.0091, 0.0057, 0.0027, 0.0042, 0.0024, 0.0076, 0.002, 0.0135, 0.0027, 0.0104, 0.0052, 0.0036, 0.0023, 0.0029, 0.0069, 0.0016, 0.0075, 0.0029, 0.0034, 0.0027, 0.0025, 0.0025, 0.003, 0.0027, 0.0035, 0.0029, 0.0016, 0.0047, 0.0053, 0.0045, 0.0025, 0.0052, 0.0037, 0.004, 0.0043, 0.0037, 0.005, 0.0042, 0.003, 0.0029, 0.0032, 0.0026, 0.0022, 0.0025, 0.0022, 0.0018, 0.002, 0.0024, 0.0028, 0.0026, 0.0038, 0.0029, 0.0032, 0.0031, 0.0019, 0.0025, 0.0017, 0.0017, 0.0026, 0.002, 0.0027, 0.0018, 0.0021, 0.0017, 0.0023, 0.0024, 0.0018, 0.0014, 0.0024, 0.0018, 0.0016, 0.0022, 0.002, 0.002, 0.0018, 0.0022, 0.0019, 0.0018, 0.0018, 0.0017, 0.0022, 0.0014, 0.0016, 0.0016, 0.0016, 0.0013, 0.0014, 0.0012, 0.0016, 0.0014, 0.0009, 0.0016, 0.0016, 0.0013, 0.0014, 0.0008, 0.0012, 0.0014, 0.0013, 0.0011, 0.0011, 0.0015, 0.0012, 0.0012, 0.0016, 0.0012, 0.0013, 0.0009, 0.0012, 0.0009, 0.0009, 0.0009, 0.0011, 0.001, 0.0011, 0.0011, 0.0012, 0.0009, 0.0013, 0.0013, 0.0011, 0.0011, 0.0008, 0.0011, 0.0011, 0.001, 0.0013, 0.001, 0.0011, 0.001, 0.0009, 0.0011, 0.0012, 0.0008, 0.0009, 0.001, 0.0013, 0.0008, 0.001, 0.0009, 0.0007, 0.0008, 0.001, 0.001, 0.0007, 0.0009, 0.0008, 0.0009, 0.0009, 0.001, 0.0008, 0.0007, 0.0008, 0.0007, 0.0009, 0.0007, 0.0007, 0.0008, 0.0008, 0.0009, 0.0008, 0.001, 0.0011, 0.0008, 0.0008, 0.0008, 0.0012, 0.0007, 0.0008, 0.0007, 0.0009, 0.0007, 0.0007, 0.0007, 0.0008, 0.0007, 0.0007, 0.0007, 0.0006, 0.0006, 0.0007, 0.0007, 0.0006, 0.0008, 0.0006, 0.0008, 0.0006, 0.0005, 0.0007, 0.0006, 0.0006, 0.0007, 0.0007, 0.0007, 0.0006, 0.0005, 0.0006, 0.0007, 0.0006, 0.0006, 0.0005, 0.0006, 0.0005, 0.0007, 0.0005, 0.0007, 0.0006, 0.0005, 0.0004, 0.0006, 0.0008, 0.0005, 0.0006, 0.0006, 0.0005, 0.0005, 0.0006, 0.0008, 0.0006, 0.0006, 0.0005, 0.0005, 0.0006, 0.0005, 0.0005, 0.0005, 0.0005, 0.0004, 0.0005, 0.0004, 0.0005, 0.0005, 0.0004, 0.0005, 0.0004, 0.0004, 0.0007, 0.0004, 0.0005, 0.0004, 0.0005, 0.0004, 0.0004, 0.0005, 0.0004, 0.0004, 0.0004, 0.0005, 0.0004, 0.0004, 0.0004, 0.0004, 0.0004, 0.0004, 0.0005, 0.0004, 0.0005, 0.0004, 0.0004, 0.0004, 0.0005, 0.0004, 0.0005, 0.0004, 0.0003, 0.0004, 0.0004, 0.0003, 0.0004, 0.0004, 0.0003, 0.0003, 0.0004, 0.0003, 0.0004, 0.0003, 0.0004, 0.0003, 0.0003, 0.0003, 0.0003, 0.0004, 0.0003, 0.0003, 0.0003, 0.0003, 0.0003, 0.0002, 0.0002, 0.0002, 0.0002]), prediction=0.0)"
      ]
     },
     "execution_count": 103,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# Returns the first ``n`` rows\n",
    "results.head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 104,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "6851"
      ]
     },
     "execution_count": 104,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# Total number of correctly classified instances\n",
    "results.where(results.label == results.prediction).count()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 105,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "19978"
      ]
     },
     "execution_count": 105,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# Total number of instances\n",
    "results.count()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 106,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "0.34292721994193615"
      ]
     },
     "execution_count": 106,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# Accuracy = total number of correctly classified instances / total number of instances\n",
    "(results.where(results.label == results.prediction).count())/results.count()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 107,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Evaluator for Multiclass Classification, which expects two input columns: prediction and label\n",
    "evaluator = MulticlassClassificationEvaluator(predictionCol='prediction', labelCol='label', metricName='accuracy')\n",
    "\n",
    "# Evaluates the output with optional parameters\n",
    "accuracy = evaluator.evaluate(results)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 108,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "0.34292721994193615"
      ]
     },
     "execution_count": 108,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# LogisticRegression accuracy (total number of correctly classified instances out of the total number of instances)\n",
    "accuracy"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### K-Fold Cross Validation"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Step 3. Tune Model\n",
    "\n",
    "On the first 80% of the data find the most accurate logistic regression model using 3-fold cross-validation with the following parameter grid:\n",
    "\n",
    "- CountVectorizer vocabulary size: `[1000, 2000]`\n",
    "- LogisticRegression regularization parameter: `[0.0, 0.1]`\n",
    "- LogisticRegression max Iteration number: `[10]`"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 109,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Builder for a param grid used in grid search-based model selection\n",
    "# Sets the given parameters in this grid to fixed values\n",
    "paramGrid = ParamGridBuilder() \\\n",
    "    .addGrid(cv.vocabSize, [1000, 2000]) \\\n",
    "    .addGrid(logreg.regParam, [0.0, 0.1]) \\\n",
    "    .build()\n",
    "\n",
    "# K-fold cross validation performs model selection by splitting the dataset into a set of non-overlapping randomly\n",
    "#   partitioned folds which are used as separate training and test datasets \n",
    "#   e.g., with k=3 folds, K-fold cross validation will generate 3 (training, test) dataset pairs, each of which\n",
    "#   uses 2/3 of the data for training and 1/3 for testing\n",
    "# Each fold is used as the test set exactly once\n",
    "crossval = CrossValidator(estimator=pipeline,\n",
    "                          estimatorParamMaps=paramGrid,\n",
    "                          evaluator=MulticlassClassificationEvaluator(),\n",
    "                          numFolds=3)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 110,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Fits a model to the input dataset with optional parameters\n",
    "cvModel = crossval.fit(train)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 111,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "[0.30080930368241526,\n",
       " 0.23247357061619897,\n",
       " 0.3280572469357257,\n",
       " 0.2580292235553692]"
      ]
     },
     "execution_count": 111,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# Average performance of parameter grid\n",
    "cvModel.avgMetrics"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Step 4: Compute Accuracy of Best Model"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 112,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Transforms the input dataset with optional parameters\n",
    "results = cvModel.transform(test)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 113,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "7193"
      ]
     },
     "execution_count": 113,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# Total number of correctly classified instances\n",
    "results.where(results.label == results.prediction).count()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 114,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "19978"
      ]
     },
     "execution_count": 114,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# Total number of instances\n",
    "results.count()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 115,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "0.3600460506557213"
      ]
     },
     "execution_count": 115,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# Accuracy = total number of correctly classified instances / total number of instances\n",
    "(results.where(results.label == results.prediction).count())/results.count()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 116,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Evaluator for Multiclass Classification, which expects two input columns: prediction and label\n",
    "evaluator = MulticlassClassificationEvaluator(predictionCol='prediction', labelCol='label', metricName='accuracy')\n",
    "\n",
    "# Evaluates the output with optional parameters\n",
    "accuracy = evaluator.evaluate(results)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 117,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "0.3600460506557213"
      ]
     },
     "execution_count": 117,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# LogisticRegression accuracy (total number of correctly classified instances out of the total number of instances)\n",
    "accuracy"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.7.4"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}