harvard-capstone-amazon-reviews-polarity.tex

% Options for packages loaded elsewhere
\PassOptionsToPackage{unicode}{hyperref}
\PassOptionsToPackage{hyphens}{url}
%
\documentclass[
]{article}
\usepackage{amsmath,amssymb}
\usepackage{lmodern}
\usepackage{iftex}
\ifPDFTeX
  \usepackage[T1]{fontenc}
  \usepackage[utf8]{inputenc}
  \usepackage{textcomp} % provide euro and other symbols
\else % if luatex or xetex
  \usepackage{unicode-math}
  \defaultfontfeatures{Scale=MatchLowercase}
  \defaultfontfeatures[\rmfamily]{Ligatures=TeX,Scale=1}
\fi
% Use upquote if available, for straight quotes in verbatim environments
\IfFileExists{upquote.sty}{\usepackage{upquote}}{}
\IfFileExists{microtype.sty}{% use microtype if available
  \usepackage[]{microtype}
  \UseMicrotypeSet[protrusion]{basicmath} % disable protrusion for tt fonts
}{}
\makeatletter
\@ifundefined{KOMAClassName}{% if non-KOMA class
  \IfFileExists{parskip.sty}{%
    \usepackage{parskip}
  }{% else
    \setlength{\parindent}{0pt}
    \setlength{\parskip}{6pt plus 2pt minus 1pt}}
}{% if KOMA class
  \KOMAoptions{parskip=half}}
\makeatother
\usepackage{xcolor}
\IfFileExists{xurl.sty}{\usepackage{xurl}}{} % add URL line breaks if available
\IfFileExists{bookmark.sty}{\usepackage{bookmark}}{\usepackage{hyperref}}
\hypersetup{
  pdftitle={Amazon Reviews Sentiment Analysis/Text Classification Choose Your Own Project A Harvard Capstone Project},
  pdfauthor={Manoj Bijoor},
  hidelinks,
  pdfcreator={LaTeX via pandoc}}
\urlstyle{same} % disable monospaced font for URLs
\usepackage[margin=1in]{geometry}
\usepackage{color}
\usepackage{fancyvrb}
\newcommand{\VerbBar}{|}
\newcommand{\VERB}{\Verb[commandchars=\\\{\}]}
\DefineVerbatimEnvironment{Highlighting}{Verbatim}{commandchars=\\\{\}}
% Add ',fontsize=\small' for more characters per line
\newenvironment{Shaded}{}{}
\newcommand{\AlertTok}[1]{\textcolor[rgb]{1.00,0.00,0.00}{\textbf{#1}}}
\newcommand{\AnnotationTok}[1]{\textcolor[rgb]{0.38,0.63,0.69}{\textbf{\textit{#1}}}}
\newcommand{\AttributeTok}[1]{\textcolor[rgb]{0.49,0.56,0.16}{#1}}
\newcommand{\BaseNTok}[1]{\textcolor[rgb]{0.25,0.63,0.44}{#1}}
\newcommand{\BuiltInTok}[1]{#1}
\newcommand{\CharTok}[1]{\textcolor[rgb]{0.25,0.44,0.63}{#1}}
\newcommand{\CommentTok}[1]{\textcolor[rgb]{0.38,0.63,0.69}{\textit{#1}}}
\newcommand{\CommentVarTok}[1]{\textcolor[rgb]{0.38,0.63,0.69}{\textbf{\textit{#1}}}}
\newcommand{\ConstantTok}[1]{\textcolor[rgb]{0.53,0.00,0.00}{#1}}
\newcommand{\ControlFlowTok}[1]{\textcolor[rgb]{0.00,0.44,0.13}{\textbf{#1}}}
\newcommand{\DataTypeTok}[1]{\textcolor[rgb]{0.56,0.13,0.00}{#1}}
\newcommand{\DecValTok}[1]{\textcolor[rgb]{0.25,0.63,0.44}{#1}}
\newcommand{\DocumentationTok}[1]{\textcolor[rgb]{0.73,0.13,0.13}{\textit{#1}}}
\newcommand{\ErrorTok}[1]{\textcolor[rgb]{1.00,0.00,0.00}{\textbf{#1}}}
\newcommand{\ExtensionTok}[1]{#1}
\newcommand{\FloatTok}[1]{\textcolor[rgb]{0.25,0.63,0.44}{#1}}
\newcommand{\FunctionTok}[1]{\textcolor[rgb]{0.02,0.16,0.49}{#1}}
\newcommand{\ImportTok}[1]{#1}
\newcommand{\InformationTok}[1]{\textcolor[rgb]{0.38,0.63,0.69}{\textbf{\textit{#1}}}}
\newcommand{\KeywordTok}[1]{\textcolor[rgb]{0.00,0.44,0.13}{\textbf{#1}}}
\newcommand{\NormalTok}[1]{#1}
\newcommand{\OperatorTok}[1]{\textcolor[rgb]{0.40,0.40,0.40}{#1}}
\newcommand{\OtherTok}[1]{\textcolor[rgb]{0.00,0.44,0.13}{#1}}
\newcommand{\PreprocessorTok}[1]{\textcolor[rgb]{0.74,0.48,0.00}{#1}}
\newcommand{\RegionMarkerTok}[1]{#1}
\newcommand{\SpecialCharTok}[1]{\textcolor[rgb]{0.25,0.44,0.63}{#1}}
\newcommand{\SpecialStringTok}[1]{\textcolor[rgb]{0.73,0.40,0.53}{#1}}
\newcommand{\StringTok}[1]{\textcolor[rgb]{0.25,0.44,0.63}{#1}}
\newcommand{\VariableTok}[1]{\textcolor[rgb]{0.10,0.09,0.49}{#1}}
\newcommand{\VerbatimStringTok}[1]{\textcolor[rgb]{0.25,0.44,0.63}{#1}}
\newcommand{\WarningTok}[1]{\textcolor[rgb]{0.38,0.63,0.69}{\textbf{\textit{#1}}}}
\usepackage{longtable,booktabs,array}
\usepackage{calc} % for calculating minipage widths
% Correct order of tables after \paragraph or \subparagraph
\usepackage{etoolbox}
\makeatletter
\patchcmd\longtable{\par}{\if@noskipsec\mbox{}\fi\par}{}{}
\makeatother
% Allow footnotes in longtable head/foot
\IfFileExists{footnotehyper.sty}{\usepackage{footnotehyper}}{\usepackage{footnote}}
\makesavenoteenv{longtable}
\usepackage{graphicx}
\makeatletter
\def\maxwidth{\ifdim\Gin@nat@width>\linewidth\linewidth\else\Gin@nat@width\fi}
\def\maxheight{\ifdim\Gin@nat@height>\textheight\textheight\else\Gin@nat@height\fi}
\makeatother
% Scale images if necessary, so that they will not overflow the page
% margins by default, and it is still possible to overwrite the defaults
% using explicit options in \includegraphics[width, height, ...]{}
\setkeys{Gin}{width=\maxwidth,height=\maxheight,keepaspectratio}
% Set default figure placement to htbp
\makeatletter
\def\fps@figure{htbp}
\makeatother
% Make links footnotes instead of hotlinks:
\DeclareRobustCommand{\href}[2]{#2\footnote{\url{#1}}}
\setlength{\emergencystretch}{3em} % prevent overfull lines
\providecommand{\tightlist}{%
  \setlength{\itemsep}{0pt}\setlength{\parskip}{0pt}}
\setcounter{secnumdepth}{5}
\usepackage[utf8]{inputenc} \usepackage[english]{babel} \usepackage{bookmark} \usepackage[]{hyperref} \hypersetup{ backref, pdftitle={"Amazon Review Polarity Harvard Capstone"}, bookmarks=true, bookmarksnumbered=true, bookmarksopen=true, bookmarksopenlevel=3, pdfpagemode=FullScreen, pdfstartpage=1, hyperindex=true, pageanchor=true, colorlinks=true, linkcolor=blue, filecolor=magenta, urlcolor=cyan } \usepackage{amsmath} \usepackage{pdflscape} \usepackage[titles]{tocloft} \usepackage{tocloft} \usepackage{titlesec} \usepackage{longtable} \usepackage{xpatch} \usepackage[T1]{fontenc} \usepackage{imakeidx} \makeindex[columns=3, title=Alphabetical Index, intoc]
\usepackage{subfig}
\usepackage{booktabs}
\usepackage{longtable}
\usepackage{array}
\usepackage{multirow}
\usepackage{wrapfig}
\usepackage{float}
\usepackage{colortbl}
\usepackage{pdflscape}
\usepackage{tabu}
\usepackage{threeparttable}
\usepackage{threeparttablex}
\usepackage[normalem]{ulem}
\usepackage{makecell}
\usepackage{xcolor}
\ifLuaTeX
  \usepackage{selnolig}  % disable illegal ligatures
\fi

\title{Amazon Reviews\\
Sentiment Analysis/Text Classification\\
Choose Your Own Project\\
A Harvard Capstone Project}
\author{Manoj Bijoor}
\date{July 28, 2021}

\begin{document}
\maketitle

\bookmark[dest=TitlePage]{Title Page}

\pagenumbering{roman}

\newpage

\newpage

\begin{center}

\hypertarget{Abstract}{}
\large{Abstract}
\bookmark[dest=Abstract]{Abstract}

\end{center}

\bigskip

Deriving truth and insight from a pile of data is a powerful but
error-prone job.

This project offers an empirical exploration on the use of Neural
networks for text classification using the Amazon Reviews Polarity
dataset.

Text classification algorithms are at the heart of a variety of software
systems that process text data at scale.

One common type of text classification is sentiment analysis, whose goal
is to identify the polarity of text content: the type of opinion it
expresses. This can take the form of a binary like/dislike rating, or a
more granular set of options, such as a star rating from 1 to 5.
Examples of sentiment analysis include analyzing Twitter posts to
determine if people liked the Black Panther movie, or extrapolating the
general public's opinion of a new brand of Nike shoes from Walmart
reviews.

Algorithms such as regularized linear models, support vector machines,
and naive Bayes models are used to predict outcomes from predictors
including text data. These algorithms use a shallow (single) mapping. In
contrast, Deep learning models approach the same tasks and have the same
goals, but the algorithms involved are different. Deep learning models
are ``deep'' in the sense that they use multiple layers to learn how to
map from input features to output outcomes.

Deep learning models can be effective for text prediction problems
because they use these multiple layers to capture complex relationships
in language.

The layers in a deep learning model are connected in a network and these
models are called Neural Networks.

Neural language models (or continuous space language models) use
continuous representations or embeddings of words to make their
predictions. These models make use of Neural networks.

Continuous space embeddings help to alleviate the curse of
dimensionality in language modeling: as language models are trained on
larger and larger texts, the number of unique words (the vocabulary)
increases. The number of possible sequences of words increases
exponentially with the size of the vocabulary, causing a data sparsity
problem because of the exponentially many sequences. Thus, statistics
are needed to properly estimate probabilities. Neural networks avoid
this problem by representing words in a distributed way, as non-linear
combinations of weights in a neural net.

Instead of using neural net language models to produce actual
probabilities, it is common to instead use the distributed
representation encoded in the networks' ``hidden'' layers as
representations of words; each word is then mapped onto an n-dimensional
real vector called the word embedding, where n is the size of the layer
just before the output layer. An alternate description is that a neural
net approximates the language function and models semantic relations
between words as linear combinations, capturing a form of
compositionality.

In this project we will cover four network architectures, namely DNN,
CNN, sepCNN and BERT. We will also first implement a Baseline linear
classifier model which serves the purpose of comparison with the deep
learning techniques.

For metrics we will use the default performance parameters for binary
classification which are Accuracy, Loss and ROC AUC (area under the
receiver operator characteristic curve).

\newpage 
\clearpage
\phantomsection
\setcounter{secnumdepth}{5}
\setcounter{tocdepth}{5}

\cleardoublepage  \hypertarget{toc}{}
\bookmark[dest=toc,level=chapter]{\contentsname} \tableofcontents

\clearpage

\newpage
\clearpage
\phantomsection

\hypertarget{list-of-tables}{%
\section*{List of tables}\label{list-of-tables}}
\addcontentsline{toc}{section}{List of tables}

\renewcommand{\listtablename}{}

\listoftables
\clearpage

\newpage
\clearpage
\phantomsection

\hypertarget{list-of-figures}{%
\section*{List of figures}\label{list-of-figures}}
\addcontentsline{toc}{section}{List of figures}

\renewcommand{\listfigurename}{}

\listoffigures
\clearpage

\newpage
\clearpage
\phantomsection
\newcommand{\listequationsname}{List of Equations}
\newlistof{equations}{equ}{List of Equations}
\newcommand{\equations}[1]{%
\refstepcounter{equations}
\addcontentsline{equ}{equations}{ \protect\numberline{\theequations}#1}\par}
\xpretocmd{\listofequations}{\addcontentsline{toc}{section}{List of Equations}}{}{}

\renewcommand{\listequationsname}{}

\listofequations
\clearpage

\newpage

\pagenumbering{arabic}

\newpage

\hypertarget{project-overview-amazon-reviews-polarity}{%
\section{Project Overview: Amazon Reviews
Polarity}\label{project-overview-amazon-reviews-polarity}}

\hypertarget{introduction}{%
\subsection{Introduction}\label{introduction}}

Deriving truth and insight from a pile of data is a powerful but
error-prone job.

Text classification algorithms are at the heart of a variety of software
systems that process text data at scale.

One common type of text classification is sentiment analysis, whose goal
is to identify the polarity of text content: the type of opinion it
expresses. This can take the form of a binary like/dislike rating, or a
more granular set of options, such as a star rating from 1 to 5.
Examples of sentiment analysis include analyzing Twitter posts to
determine if people liked the Black Panther movie, or extrapolating the
general public's opinion of a new brand of Nike shoes from Walmart
reviews.

Algorithms such as regularized linear models, support vector machines,
and naive Bayes models are used to predict outcomes from predictors
including text data. These algorithms use a shallow (single) mapping. In
contrast, Deep learning models approach the same tasks and have the same
goals, but the algorithms involved are different. Deep learning models
are ``deep'' in the sense that they use multiple layers to learn how to
map from input features to output outcomes.

Deep learning models can be effective for text prediction problems
because they use these multiple layers to capture complex relationships
in language.

The layers in a deep learning model are connected in a network and these
models are called neural networks.

\hypertarget{neural-networks}{%
\subsubsection{Neural networks}\label{neural-networks}}

Neural language models (or continuous space language models) use
continuous representations or
\href{https://en.wikipedia.org/wiki/Word_embedding}{embeddings of words}
to make their
predictions.\href{https://karpathy.github.io/2015/05/21/rnn-effectiveness/}{Karpathy,
Andrej. ``The Unreasonable Effectiveness of Recurrent Neural Networks''}
These models make use of
\href{https://en.wikipedia.org/wiki/Artificial_neural_network}{Neural
networks}.

Continuous space embeddings help to alleviate the
\href{https://en.wikipedia.org/wiki/Curse_of_dimensionality}{curse of
dimensionality} in language modeling: as language models are trained on
larger and larger texts, the number of unique words (the vocabulary)
increases.\href{https://en.wikipedia.org/wiki/Heaps\%27_law}{Heaps'
law}. The number of possible sequences of words increases exponentially
with the size of the vocabulary, causing a data sparsity problem because
of the exponentially many sequences. Thus, statistics are needed to
properly estimate probabilities. Neural networks avoid this problem by
representing words in a distributed way, as non-linear combinations of
weights in a neural
net.\href{https://ui.adsabs.harvard.edu/abs/2008SchpJ...3.3881B/abstract}{Bengio,
Yoshua (2008). ``Neural net language models''. Scholarpedia. 3. p.~3881.
Bibcode:2008SchpJ\ldots3.3881B. doi:10.4249/scholarpedia.3881} An
alternate description is that a neural net approximates the language
function.

Instead of using neural net language models to produce actual
probabilities, it is common to instead use the distributed
representation encoded in the networks' ``hidden'' layers as
representations of words;\\
A hidden layer is a synthetic layer in a neural network between the
input layer (that is, the features) and the output layer (the
prediction). Hidden layers typically contain an activation function such
as
\href{https://developers.google.com/machine-learning/glossary?utm_source=DevSite\&utm_campaign=Text-Class-Guide\&utm_medium=referral\&utm_content=glossary\&utm_term=sepCNN\#rectified-linear-unit-relu}{ReLU}
for training. A deep neural network contains more than one hidden layer.
Each word is then mapped onto an n-dimensional real vector called the
word embedding, where n is the size of the layer just before the output
layer. The representations in skip-gram models for example have the
distinct characteristic that they model semantic relations between words
as \href{https://en.wikipedia.org/wiki/Linear_combination}{linear
combinations}, capturing a form of
\href{https://en.wikipedia.org/wiki/Principle_of_compositionality}{compositionality}.

In this project we will cover four network architectures, namely:

\begin{enumerate}
\def\labelenumi{\arabic{enumi}.}
\item
  DNN - Dense Neural Network - a bridge between the ``shallow'' learning
  approaches and the other 3 - CNN, sepCNN, BERT.
\item
  CNN - Convolutional Neural Network - advanced architecture appropriate
  for text data because they can capture specific local patterns.
\item
  sepCNN - Depthwise Separable Convolutional Neural Network.
\item
  BERT - Bidirectional Encoder Representations from Transformers.
\end{enumerate}

We will also first implement a Baseline linear classifier model which
serves the purpose of comparison with the deep learning techniques we
will implement later on, and also as a succinct summary of a basic
supervised machine learning analysis for text.

This linear baseline is a regularized linear model trained on the same
data set, using tf-idf weights and 5000 tokens.

For metrics we will use the default performance parameters for binary
classification which are Accuracy, Loss and ROC AUC (area under the
receiver operator characteristic curve).

We will also use the confusion matrix to get an overview of our model
performance, as it includes rich information.

We will use tidymodels packages along with Tensorflow, the R interface
to Keras. See \href{https://CRAN.R-project.org/package=keras}{Allaire,
JJ, and François Chollet. 2021. keras: R Interface to 'Keras'} for
preprocessing, modeling, and evaluation, and
\href{https://www.tidytextmining.com/}{Silge, Julia, and David Robinson.
2017. Text Mining with R: A Tidy Approach. 1st ed.~O'Reilly Media,
Inc.}, \href{https://smltar.com/}{Supervised Machine Learning for Text
Analysis in R, by Emil Hvitfeldt and Julia Silge.} and
\href{https://www.tmwr.org/}{Tidy Modeling with R, Max Kuhn and Julia
Silge, Version 0.0.1.9010, 2021-07-19} and how can we forget
\href{https://rafalab.github.io/dsbook/}{Introduction to Data Science,
Data Analysis and Prediction Algorithms with R - Rafael A. Irizarry,
2021-07-03}.

The keras R package provides an interface for R users to Keras, a
high-level API for building neural networks.

This project will use some key machine learning best practices for
solving text classification problems.\\
Here's what you'll learn:

\begin{enumerate}
\def\labelenumi{\arabic{enumi}.}
\tightlist
\item
  The high-level, end-to-end workflow for solving text classification
  problems using machine learning
\item
  How to choose the right model for your text classification problem
\item
  How to implement your model of choice using TensorFlow with Keras
  acting as an interface for the TensorFlow library
\end{enumerate}

I have used/mentioned several references throughout the project.

This project depends on python and R software for Tensorflow and Keras
that needs to be installed both inside and outside of R. As each
individual's environment may be different, I cannot automate this part
in my code.

R side:\\
\url{https://cran.r-project.org/}\strut \\
\url{https://tensorflow.rstudio.com/installation/}\strut \\
\url{https://tensorflow.rstudio.com/installation/gpu/local_gpu/}

Python side:\\
\url{https://www.tensorflow.org/install}\strut \\
\url{https://www.anaconda.com/products/individual}\strut \\
\url{https://keras.io/}

Instead of cluttering code with comments, I ask you to please use these
references and the rstudio help (?cmd/??cmd) if you are not very
familiar with any specific command. Most commands are pretty self
explanatory if you are even a little familiar with R.

Here are some more references:

\hypertarget{references}{%
\subsection{References}\label{references}}

\href{https://www.tensorflow.org/}{Tensorflow} is an end-to-end open
source platform for machine learning. It has a comprehensive, flexible
ecosystem of tools, libraries and community resources that lets
researchers push the state-of-the-art in ML and developers easily build
and deploy ML powered applications.

The \href{https://tfhub.dev/}{TensorFlow Hub} lets you search and
discover hundreds of trained, ready-to-deploy machine learning models in
one place.

\href{https://tensorflow.rstudio.com/}{Tensorflow for R} provides an R
interface for Tensorflow.

\href{https://www.tmwr.org/}{Tidy Modeling with R}

\href{https://yihui.org/tinytex/}{Tinytex}\\
I have used tinytex in code chunks.

\href{https://www.overleaf.com/learn/latex}{Latex}\\
I have used Latex beyond the very basic provided by default templates in
RStudio. Too numerous to explain. Though that much is not needed, I have
used it to learn and make better pdf docs.

\href{https://bookdown.org/yihui/rmarkdown}{Rmarkdown}

\newpage

\hypertarget{text-classification-workflow}{%
\subsection{Text Classification
Workflow}\label{text-classification-workflow}}

Here's a high-level overview of the workflow used to solve machine
learning problems:

Step 1: Gather Data\\
Step 2: Explore Your Data\\
Step 2.5: Choose a Model*\\
Step 3: Prepare Your Data\\
Step 4: Build, Train, and Evaluate Your Model\\
Step 5: Tune Hyperparameters\\
Step 6: Deploy Your Model

The following sections explain each step in detail, and how to implement
them for text data.

\hypertarget{gather-data}{%
\subsubsection{Gather Data}\label{gather-data}}

Gathering data is the most important step in solving any supervised
machine learning problem. Your text classifier can only be as good as
the dataset it is built from.

Here are some important things to remember when collecting data:

\begin{enumerate}
\def\labelenumi{\arabic{enumi}.}
\item
  If you are using a public API, understand the limitations of the API
  before using them. For example, some APIs set a limit on the rate at
  which you can make queries.
\item
  The more training examples/samples you have, the better. This will
  help your model generalize better.
\item
  Make sure the number of samples for every class or topic is not overly
  imbalanced. That is, you should have comparable number of samples in
  each class.
\item
  Make sure that your samples adequately cover the space of possible
  inputs, not only the common cases.
\end{enumerate}

This dataset contains amazon reviews posted by people on the Amazon
website, and is a classic example of a sentiment analysis problem.

Amazon Review Polarity Dataset - Version 3, Updated 09/09/2015

ORIGIN

The Amazon reviews dataset consists of reviews from amazon. The data
span a period of 18 years, including \textasciitilde35 million reviews
up to March 2013. Reviews include product and user information, ratings,
and a plaintext review. For more information, please refer to the
following paper:
\href{https://cs.stanford.edu/people/jure/pubs/reviews-recsys13.pdf}{J.
McAuley and J. Leskovec. Hidden factors and hidden topics: Understanding
rating dimensions with review text. In Proceedings of the 7th ACM
Conference on Recommender Systems, RecSys '13, pages 165--172, New York,
NY, USA, 2013. ACM}.

The Amazon reviews polarity dataset was constructed by Xiang Zhang
(\href{mailto:xiang.zhang@nyu.edu}{\nolinkurl{xiang.zhang@nyu.edu}})
from the above dataset. It is used as a text classification benchmark in
the following paper: Xiang Zhang, Junbo Zhao, Yann LeCun.
\href{https://arxiv.org/abs/1509.01626}{Character-level Convolutional
Networks for Text Classification}. Advances in Neural Information
Processing Systems 28 (NIPS 2015).

Here is an Abstract of that paper:

This article offers an empirical exploration on the use of
character-level convolutional networks (ConvNets) for text
classification. We constructed several large-scale datasets to show that
character-level convolutional networks could achieve state-of-the-art or
competitive results. Comparisons are offered against traditional models
such as bag of words, n-grams and their TFIDF variants, and deep
learning models such as word-based ConvNets and recurrent neural
networks.

Coming back to our project: As Google has changed it's API, I had to
download the dataset manually from the following URL:

Please select file named ``amazon\_review\_polarity\_csv.tar.gz'' and
download it to the project directory.

Download Location URL :
\href{https://drive.google.com/drive/folders/0Bz8a_Dbh9Qhbfll6bVpmNUtUcFdjYmF2SEpmZUZUcVNiMUw1TWN6RDV3a0JHT3kxLVhVR2M?resourcekey=0-TLwzfR2O-D2aPitmn5o9VQ}{Xiang
Zhang Google Drive}

DESCRIPTION

The Amazon reviews polarity dataset is constructed by taking review
score 1 and 2 as negative, and 4 and 5 as positive. Samples of score 3
is ignored. In the dataset, class 1 is the negative and class 2 is the
positive. Each class has 1,800,000 training samples and 200,000 testing
samples.

The files train.csv and test.csv contain all the training samples as
comma-separated values. There are 3 columns in them, corresponding to
label/class index (1 or 2), review title and review text. The review
title and text are escaped using double quotes (``), and any internal
double quote is escaped by 2 double quotes (''``). New lines are escaped
by a backslash followed with an''n'' character, that is
``\textbackslash n''.

\newpage

\hypertarget{explore-your-data}{%
\subsubsection{Explore Your Data}\label{explore-your-data}}

Building and training a model is only one part of the workflow.
Understanding the characteristics of your data beforehand will enable
you to build a better model. This could simply mean obtaining a higher
accuracy. It could also mean requiring less data for training, or fewer
computational resources.

\hypertarget{load-the-dataset}{%
\paragraph{Load the Dataset}\label{load-the-dataset}}

First up, let's load the dataset into R.

In the dataset, class 1 is the negative and class 2 is the positive
review. We will change these to 0 and 1.

columns = (0, 1, 2) \# 0 - label/class index, 1 - title/subject, 2 -
text body/review.

In this project we will NOT be using the ``title'' data. We will use
only ``label'' and ``text''. Also note that I have more comments in the
code file/s than in the pdf document.

\begin{Shaded}
\begin{Highlighting}[]
\FunctionTok{untar}\NormalTok{(}\StringTok{"amazon\_review\_polarity\_csv.tar.gz"}\NormalTok{, }\AttributeTok{list =} \ConstantTok{TRUE}\NormalTok{)  }\DocumentationTok{\#\# check contents}
\NormalTok{[}\DecValTok{1}\NormalTok{] }\StringTok{"amazon\_review\_polarity\_csv/"}          
\NormalTok{[}\DecValTok{2}\NormalTok{] }\StringTok{"amazon\_review\_polarity\_csv/test.csv"}  
\NormalTok{[}\DecValTok{3}\NormalTok{] }\StringTok{"amazon\_review\_polarity\_csv/train.csv"} 
\NormalTok{[}\DecValTok{4}\NormalTok{] }\StringTok{"amazon\_review\_polarity\_csv/readme.txt"}
\FunctionTok{untar}\NormalTok{(}\StringTok{"amazon\_review\_polarity\_csv.tar.gz"}\NormalTok{)}
\end{Highlighting}
\end{Shaded}

\newpage

\hypertarget{check-the-data}{%
\paragraph{Check the Data}\label{check-the-data}}

After loading the data, it's good practice to run some checks on it:
pick a few samples and manually check if they are consistent with your
expectations. For example see Table \ref{tbl:amazon_train}

\begin{Shaded}
\begin{Highlighting}[]
\FunctionTok{glimpse}\NormalTok{(amazon\_train)}
\NormalTok{Rows}\SpecialCharTok{:} \DecValTok{2}\NormalTok{,}\DecValTok{879}\NormalTok{,}\DecValTok{960}
\NormalTok{Columns}\SpecialCharTok{:} \DecValTok{3}
\SpecialCharTok{$}\NormalTok{ label }\SpecialCharTok{\textless{}}\NormalTok{dbl}\SpecialCharTok{\textgreater{}} \DecValTok{0}\NormalTok{, }\DecValTok{1}\NormalTok{, }\DecValTok{0}\NormalTok{, }\DecValTok{1}\NormalTok{, }\DecValTok{1}\NormalTok{, }\DecValTok{1}\NormalTok{, }\DecValTok{0}\NormalTok{, }\DecValTok{0}\NormalTok{, }\DecValTok{1}\NormalTok{, }\DecValTok{1}\NormalTok{, }\DecValTok{1}\NormalTok{, }\DecValTok{1}\NormalTok{, }\DecValTok{1}\NormalTok{, }\DecValTok{0}\NormalTok{, }\DecValTok{0}\NormalTok{, }\DecValTok{1}\NormalTok{, }\DecValTok{1}\NormalTok{, }\DecValTok{1}\NormalTok{, }\DecValTok{0}\NormalTok{, }\DecValTok{0}\NormalTok{, }\DecValTok{1}\NormalTok{, }\DecValTok{0}\SpecialCharTok{\textasciitilde{}}
\ErrorTok{$}\NormalTok{ title }\SpecialCharTok{\textless{}}\NormalTok{chr}\SpecialCharTok{\textgreater{}} \StringTok{"For Older Mac Operating Systems Only"}\NormalTok{, }\StringTok{"greener today than yest\textasciitilde{}}
\StringTok{$ text  \textless{}chr\textgreater{} "}\NormalTok{Does not work on Mac OSX Per Scholastic tech support If you hav}\SpecialCharTok{\textasciitilde{}}
\end{Highlighting}
\end{Shaded}

\begin{table}[H]

\caption{\label{tab:chk_data_2}Amazon Train  data\label{tbl:amazon_train}}
\centering
\fontsize{6}{8}\selectfont
\begin{tabular}[t]{rll}
\toprule
label & title & text\\
\midrule
0 & For Older Mac Operating Systems Only & Does not work on Mac OSX Per Scholastic tech support If you have a newer Mac machine with one of the later OS versions ( or later) the game will not work as it was meant to run on an older system\\
1 & greener today than yesterday & I was a skeptic however I nowhave three newspapers the bible six books and two magazines on my kindle I ve wowed my church reading group with skipping back and forth between our book we are reading and the bible I ve eliminated several paper subscriptions and actually am reading more because i can easily carry it all with me The page is easy on the eyes I really like it\\
0 & Overrated very overrated & This film is a disappointment Based on the reviews I read here at Amazon and in other media I expected an exciting and visually engrossing experience This is a nineteen nineties touchy feely melodrama and furthermore it s poorly done Can I get my money back\\
1 & well really & dmst are my favourite band and have been for a while now just mind blowing i wanted to make one correction to the previous review though the do makes are from toronto not quebec\\
1 & dynomax super turbo exhaust & it fit good took only about mins to put on little quiter than i wanted ( but for the price can t beat it ) it is getting a littler louder everyday i drive it starting to burn some of the glass packing out\\
1 & East LA Marine the Guy Gabaldon Story & This movie puts history in perspective for those who know the story of Guy Gabaldon I think that he is an unsung hero that should have been awarded for the lives that he saved on both sides I think that you will be glad that you watched this movie Not the Hollywood version but through this you get to meet the actual hero not Hollywood s version of who they thought he should be\\
0 & World of Bad Support & Before getting this game be sure to check out the Support forums WoW (World of warcraft) is suffering from things like Peoples accounts expiring with no way to get a CC or game card updated High graphics glitches make game unplayableHigh rate of computer lock ups freezing and Blue screening Blizzards support staff ask the users to update drivers There latest patch has caused a large amount of players to be unable to play at all So make sure that your computer won t have these issues Even though systems with gig of ram and the best video cards have issues maybe yours won t I recommended waiting for Blizzard to finish the stress test they call GOLD Instead get any other MMORPG none are having the issues this one has If you do buy it and can t play please note that for the last days Blizzards support line has been ringing fast busy hehe\\
0 & disapointing & Only two songs are great Desire All I want is you There are some good live performaces but Helter Skelter and All along the watch tower covers were very bad moves If you re a die hard fan buy this for the two songs I mentioned because they re classics but otherwise this is hardly essential\\
1 & SAITEK X FLIGHT CONTROL SYSTEM BLOWS YOU AWAY & When I purchased my Flight Simulator Deluxe Edition I chose to purchase these controls as well I wanted as real a feel as I could get with my gaming system Well at the time they were the best on the shelf that I could find Nothing else came close to these The first few reviewers have explained the controls already They are right on the money You will want to purchase these along with your game\\
1 & the secret of science & The best kept secret of science is how strongly it points towards a creator and dovetails with Christianity In this marvelously lucid book the eminent physical chemist Henry Schaefer unfolds the secret\\
\bottomrule
\end{tabular}
\end{table}

Labels : Negative reviews = 0, Positive reviews = 1

\begin{Shaded}
\begin{Highlighting}[]
\FunctionTok{unique}\NormalTok{(amazon\_train}\SpecialCharTok{$}\NormalTok{label)}
\NormalTok{[}\DecValTok{1}\NormalTok{] }\DecValTok{0} \DecValTok{1}
\end{Highlighting}
\end{Shaded}

\newpage

\hypertarget{collect-key-metrics}{%
\paragraph{Collect Key Metrics}\label{collect-key-metrics}}

Once you've verified the data, collect the following important metrics
that can help characterize your text classification problem:

1.Number of samples: Total number of examples you have in the data.

2.Number of classes: Total number of topics or categories in the data.

3.Number of samples per class: Number of samples per class
(topic/category). In a balanced dataset, all classes will have a similar
number of samples; in an imbalanced dataset, the number of samples in
each class will vary widely.

4.Number of words per sample: Median number of words in one sample.

5.Frequency distribution of words: Distribution showing the frequency
(number of occurrences) of each word in the dataset.

6.Distribution of sample length: Distribution showing the number of
words per sample in the dataset.

Number of samples

\begin{Shaded}
\begin{Highlighting}[]
\NormalTok{(num\_samples }\OtherTok{\textless{}{-}} \FunctionTok{nrow}\NormalTok{(amazon\_train))}
\NormalTok{[}\DecValTok{1}\NormalTok{] }\DecValTok{2879960}
\end{Highlighting}
\end{Shaded}

Number of classes

\begin{Shaded}
\begin{Highlighting}[]
\NormalTok{(num\_classes }\OtherTok{\textless{}{-}} \FunctionTok{length}\NormalTok{(}\FunctionTok{unique}\NormalTok{(amazon\_train}\SpecialCharTok{$}\NormalTok{label)))}
\NormalTok{[}\DecValTok{1}\NormalTok{] }\DecValTok{2}
\end{Highlighting}
\end{Shaded}

Number of samples per class

\begin{Shaded}
\begin{Highlighting}[]
\CommentTok{\# Pretty Balanced classes}
\NormalTok{(num\_samples\_per\_class }\OtherTok{\textless{}{-}}\NormalTok{ amazon\_train }\SpecialCharTok{\%\textgreater{}\%}
    \FunctionTok{count}\NormalTok{(label))}
\end{Highlighting}
\end{Shaded}

\begin{tabular}{r|r}
\hline
label & n\\
\hline
0 & 1439405\\
\hline
1 & 1440555\\
\hline
\end{tabular}

Number of words per sample

\begin{Shaded}
\begin{Highlighting}[]
\NormalTok{amazon\_train\_text\_wordCount }\OtherTok{\textless{}{-}} \FunctionTok{sapply}\NormalTok{(temp, length)}

\NormalTok{(mean\_num\_words\_per\_sample }\OtherTok{\textless{}{-}} \FunctionTok{mean}\NormalTok{(amazon\_train\_text\_wordCount))}
\NormalTok{[}\DecValTok{1}\NormalTok{] }\FloatTok{75.89602}

\NormalTok{(median\_num\_words\_per\_sample }\OtherTok{\textless{}{-}} \FunctionTok{median}\NormalTok{(amazon\_train\_text\_wordCount))}
\NormalTok{[}\DecValTok{1}\NormalTok{] }\DecValTok{67}
\end{Highlighting}
\end{Shaded}

\newpage

\hypertarget{tokenization}{%
\paragraph{Tokenization}\label{tokenization}}

To build features for supervised machine learning from natural language,
we need some way of representing raw text as numbers so we can perform
computation on them. Typically, one of the first steps in this
transformation from natural language to feature, or any of kind of text
analysis, is tokenization. Knowing what tokenization and tokens are,
along with the related concept of an n-gram, is important for almost any
natural language processing task.

Tokenization in NLP/Text Classification is essentially splitting a
phrase, sentence, paragraph, or an entire text document into smaller
units, such as individual words or terms. Each of these smaller units
are called tokens.

For Frequency distribution of words(nrams) and for Top 25 words see
Table \ref{tbl:train_words} and Figure \ref{fig:model_1}

\begin{table}

\caption{\label{tab:freq_dist_ngrams}Frequency distribution of words\label{tbl:train_words}}
\centering
\begin{tabular}[t]{lrrrr}
\toprule
word & n & total & rank & term frequency\\
\midrule
the & 11106073 & 218378699 & 1 & 0.0508569\\
i & 6544247 & 218378699 & 2 & 0.0299674\\
and & 6018678 & 218378699 & 3 & 0.0275607\\
a & 5452771 & 218378699 & 4 & 0.0249693\\
to & 5398243 & 218378699 & 5 & 0.0247196\\
it & 5028997 & 218378699 & 6 & 0.0230288\\
of & 4325341 & 218378699 & 7 & 0.0198066\\
this & 4083354 & 218378699 & 8 & 0.0186985\\
is & 3850538 & 218378699 & 9 & 0.0176324\\
in & 2594242 & 218378699 & 10 & 0.0118796\\
\bottomrule
\end{tabular}
\end{table}

\begin{verbatim}
Warning: Ignoring unknown parameters: binwidth
\end{verbatim}

\begin{figure}
\centering
\includegraphics{figures/plot_freq_dist_ngrams-1.pdf}
\caption{Frequency distribution of words(nrams) for Top 25
words\label{fig:model_1}}
\end{figure}

\newpage

\hypertarget{stopwords}{%
\paragraph{Stopwords}\label{stopwords}}

Once we have split text into tokens, it often becomes clear that not all
words carry the same amount of information, if any information at all,
for a predictive modeling task. Common words that carry little (or
perhaps no) meaningful information are called stop words. It is common
advice and practice to remove stop words for various NLP tasks.

The concept of stop words has a long history with Hans Peter Luhn
credited with coining the term in 1960.
\href{https://doi.org/10.1002/asi.5090110403}{Luhn, H. P. 1960. ``Key
Word-in-Context Index for Technical Literature (kwic Index).'' American
Documentation 11 (4): 288--295. doi:10.1002/asi.5090110403}. Examples of
these words in English are ``a,'' ``the,'' ``of,'' and ``didn't.'' These
words are very common and typically don't add much to the meaning of a
text but instead ensure the structure of a sentence is sound.

Historically, one of the main reasons for removing stop words was to
decrease the computational time for text mining; it can be regarded as a
dimensionality reduction of text data and was commonly used in search
engines to give better results
\href{https://doi.org/10.1145/1835449.1835499}{Huston, Samuel, and W.
Bruce Croft. 2010. ``Evaluating Verbose Query Processing Techniques.''
In Proceedings of the 33rd International ACM SIGIR Conference on
Research and Development in Information Retrieval, 291--298. SIGIR '10.
New York, NY, USA: ACM. doi:10.1145/1835449.1835499}.

For Frequency distribution of words(ngrams) and for Top 25 words
excluding stopwords see Table \ref{tbl:train_words_sw} and Figure
\ref{fig:model_2}

Using Pre-made stopwords

\begin{Shaded}
\begin{Highlighting}[]
\FunctionTok{length}\NormalTok{(}\FunctionTok{stopwords}\NormalTok{(}\AttributeTok{source =} \StringTok{"smart"}\NormalTok{))}
\NormalTok{[}\DecValTok{1}\NormalTok{] }\DecValTok{571}
\FunctionTok{length}\NormalTok{(}\FunctionTok{stopwords}\NormalTok{(}\AttributeTok{source =} \StringTok{"snowball"}\NormalTok{))}
\NormalTok{[}\DecValTok{1}\NormalTok{] }\DecValTok{175}
\FunctionTok{length}\NormalTok{(}\FunctionTok{stopwords}\NormalTok{(}\AttributeTok{source =} \StringTok{"stopwords{-}iso"}\NormalTok{))}
\NormalTok{[}\DecValTok{1}\NormalTok{] }\DecValTok{1298}
\end{Highlighting}
\end{Shaded}

Frequency distribution of words with stopwords removed

We will use the ``stopwords-iso'' Pre-made stopwords along with a few
unique to our case

\begin{Shaded}
\begin{Highlighting}[]
\NormalTok{mystopwords }\OtherTok{\textless{}{-}} \FunctionTok{c}\NormalTok{(}\StringTok{"s"}\NormalTok{, }\StringTok{"t"}\NormalTok{, }\StringTok{"m"}\NormalTok{, }\StringTok{"ve"}\NormalTok{, }\StringTok{"re"}\NormalTok{, }\StringTok{"d"}\NormalTok{, }\StringTok{"ll"}\NormalTok{)}
\end{Highlighting}
\end{Shaded}

\begin{table}

\caption{\label{tab:freq_dist_ngrams_stopwords}Frequency distribution of words excluding stopwords\label{tbl:train_words_sw}}
\centering
\begin{tabular}[t]{lrrrr}
\toprule
word & n & total & rank & term frequency\\
\midrule
book & 1441251 & 71253939 & 1 & 0.0202270\\
read & 513081 & 71253939 & 2 & 0.0072007\\
time & 506943 & 71253939 & 3 & 0.0071146\\
movie & 431317 & 71253939 & 4 & 0.0060532\\
love & 332012 & 71253939 & 5 & 0.0046596\\
product & 306813 & 71253939 & 6 & 0.0043059\\
bought & 292201 & 71253939 & 7 & 0.0041008\\
album & 265835 & 71253939 & 8 & 0.0037308\\
story & 264836 & 71253939 & 9 & 0.0037168\\
music & 235009 & 71253939 & 10 & 0.0032982\\
\bottomrule
\end{tabular}
\end{table}

\begin{figure}
\centering
\includegraphics{figures/plot_freq_dist_ngrams_stopwords-1.pdf}
\caption{Frequency distribution of words(nrams) for Top 25 words
excluding stopwords\label{fig:model_2}}
\end{figure}

\newpage

Here are Google's recommendations after decades of research:

Algorithm for Data Preparation and Model Building

\begin{enumerate}
\def\labelenumi{\arabic{enumi}.}
\tightlist
\item
  Calculate the number of samples/number of words per sample ratio.
\item
  If this ratio is less than 1500, tokenize the text as n-grams and use
  a simple multi-layer perceptron (MLP) model to classify them (left
  branch in the flowchart below):
\end{enumerate}

\begin{enumerate}
\def\labelenumi{\alph{enumi}.}
\tightlist
\item
  Split the samples into word n-grams; convert the n-grams into vectors.
\item
  Score the importance of the vectors and then select the top 20K using
  the scores.
\item
  Build an MLP model.
\end{enumerate}

\begin{enumerate}
\def\labelenumi{\arabic{enumi}.}
\setcounter{enumi}{2}
\tightlist
\item
  If the ratio is greater than 1500, tokenize the text as sequences and
  use a
  \href{https://developers.google.com/machine-learning/glossary?utm_source=DevSite\&utm_campaign=Text-Class-Guide\&utm_medium=referral\&utm_content=glossary\&utm_term=sepCNN\#depthwise-separable-convolutional-neural-network-sepcnn}{sepCNN}
  model to classify them (right branch in the flowchart below):
\end{enumerate}

\begin{enumerate}
\def\labelenumi{\alph{enumi}.}
\tightlist
\item
  Split the samples into words; select the top 20K words based on their
  frequency.
\item
  Convert the samples into word sequence vectors.
\item
  If the original number of samples/number of words per sample ratio is
  less than 15K, using a fine-tuned pre-trained embedding with the
  sepCNN model will likely provide the best results.
\end{enumerate}

\begin{enumerate}
\def\labelenumi{\arabic{enumi}.}
\setcounter{enumi}{3}
\tightlist
\item
  Measure the model performance with different hyperparameter values to
  find the best model configuration for the dataset.
\end{enumerate}

\begin{Shaded}
\begin{Highlighting}[]
\CommentTok{\# 3. If the ratio is greater than 1500, tokenize the text as sequences and use}
\CommentTok{\# a sepCNN model see above}

\NormalTok{(S\_W\_ratio }\OtherTok{\textless{}{-}}\NormalTok{ num\_samples}\SpecialCharTok{/}\NormalTok{median\_num\_words\_per\_sample)}
\NormalTok{[}\DecValTok{1}\NormalTok{] }\FloatTok{42984.48}
\end{Highlighting}
\end{Shaded}

\newpage

\hypertarget{preprocessing-for-deep-learning-continued-with-more-exploration}{%
\subsection{Preprocessing for deep learning continued with more
exploration}\label{preprocessing-for-deep-learning-continued-with-more-exploration}}

For ``Number of words per review text'' see Figure \ref{fig:model_3}

For ``Number of words per review title'' see Figure \ref{fig:model_4}

For ``Number of words per review text by label'' see Figure
\ref{fig:model_5}

For ``Number of words per review title by label'' see Figure
\ref{fig:model_6}

For ``Sample/Subset of our training dataset'' see Table
\ref{tbl:amazon_subset_train}

\begin{figure}
\centering
\includegraphics{figures/preproc_1-1.pdf}
\caption{Number of words per review text\label{fig:model_3}}
\end{figure}

\begin{figure}
\centering
\includegraphics{figures/preproc_2-1.pdf}
\caption{Number of words per review title\label{fig:model_4}}
\end{figure}

\begin{figure}
\centering
\includegraphics{figures/preproc_3-1.pdf}
\caption{Number of words per review text by label\label{fig:model_5}}
\end{figure}

\begin{figure}
\centering
\includegraphics{figures/preproc_4-1.pdf}
\caption{Number of words per review title by label\label{fig:model_6}}
\end{figure}

Let's trim down our training dataset due to computing resource
limitations.

\begin{Shaded}
\begin{Highlighting}[]
\NormalTok{amazon\_subset\_train }\OtherTok{\textless{}{-}}\NormalTok{ amazon\_train }\SpecialCharTok{\%\textgreater{}\%}
    \FunctionTok{select}\NormalTok{(}\SpecialCharTok{{-}}\NormalTok{title) }\SpecialCharTok{\%\textgreater{}\%}
    \FunctionTok{mutate}\NormalTok{(}\AttributeTok{n\_words =}\NormalTok{ tokenizers}\SpecialCharTok{::}\FunctionTok{count\_words}\NormalTok{(text)) }\SpecialCharTok{\%\textgreater{}\%}
    \FunctionTok{filter}\NormalTok{((n\_words }\SpecialCharTok{\textless{}} \DecValTok{35}\NormalTok{) }\SpecialCharTok{\&}\NormalTok{ (n\_words }\SpecialCharTok{\textgreater{}} \DecValTok{5}\NormalTok{)) }\SpecialCharTok{\%\textgreater{}\%}
    \FunctionTok{select}\NormalTok{(}\SpecialCharTok{{-}}\NormalTok{n\_words)}
\FunctionTok{dim}\NormalTok{(amazon\_subset\_train)}
\NormalTok{[}\DecValTok{1}\NormalTok{] }\DecValTok{579545}      \DecValTok{2}
\CommentTok{\# head(amazon\_subset\_train)}
\end{Highlighting}
\end{Shaded}

\begin{table}

\caption{\label{tab:subset_train}Sample/Subset of our training dataset\label{tbl:amazon_subset_train}}
\centering
\begin{tabular}[t]{rl}
\toprule
label & text\\
\midrule
1 & dmst are my favourite band and have been for a while now just mind blowing i wanted to make one correction to the previous review though the do makes are from toronto not quebec\\
1 & The best kept secret of science is how strongly it points towards a creator and dovetails with Christianity In this marvelously lucid book the eminent physical chemist Henry Schaefer unfolds the secret\\
1 & Our children ( age ) love these DVD s They are somewhat educational and seem to be much better than the cartoons you find on regular TV\\
1 & I have enjoyed using these picks I am a beginning guitar player and using a thin gauge pick like this makes strumming much easier\\
1 & This book is very concise and useful I found it very easy to accurately translate quickly\\
1 & Please don t deny your selfof a most irresistable chocolate This biography Am confident to sayForget what s current now Just Get this caviar of a biographyand drool on with pleasures un expected\\
0 & These are not clear Not even close They are opaque (and even closer to white) and not advertised as such To me they were completely useless Buyer beware\\
1 & Comfortable and classy but they scratch really easy Other than that a good buy if you don t plan on wearing them everyday\\
0 & I read about of the book then finally dropped it because I found it rather tiring (I had read Five Children and It before and thought I might like more book of the author\\
0 & I bought to look up the rules of Euchre and it had hardly anything here I thought the book sucked\\
\bottomrule
\end{tabular}
\end{table}

\newpage

\hypertarget{model-baseline-linear-classifier}{%
\section{Model Baseline linear
classifier}\label{model-baseline-linear-classifier}}

This model serves the purpose of comparison with the deep learning
techniques we will implement later on, and also as a succinct summary of
a basic supervised machine learning analysis for text.

This linear baseline is a regularized linear model trained on the same
data set, using tf-idf weights and 5000 tokens.

\hypertarget{modify-label-column-to-factor}{%
\subsection{Modify label column to
factor}\label{modify-label-column-to-factor}}

\begin{Shaded}
\begin{Highlighting}[]
\CommentTok{\# Free computer resources}
\FunctionTok{rm}\NormalTok{(amazon\_train, amazon\_val, amazon\_train\_text\_wordCount, num\_samples\_per\_class,}
\NormalTok{    temp, total\_words, train\_words)}
\FunctionTok{rm}\NormalTok{(mean\_num\_words\_per\_sample, median\_num\_words\_per\_sample, num\_classes, num\_samples,}
\NormalTok{    S\_W\_ratio)}
\FunctionTok{gc}\NormalTok{()}
           \FunctionTok{used}\NormalTok{  (Mb) gc }\FunctionTok{trigger}\NormalTok{   (Mb)   max }\FunctionTok{used}\NormalTok{    (Mb)}
\NormalTok{Ncells  }\DecValTok{3372585} \FloatTok{180.2}   \DecValTok{16004112}  \FloatTok{854.8}   \DecValTok{20005140}  \FloatTok{1068.4}
\NormalTok{Vcells }\DecValTok{24396993} \FloatTok{186.2} \DecValTok{1240182133} \FloatTok{9461.9} \DecValTok{1550227666} \FloatTok{11827.3}

\CommentTok{\# save(amazon\_subset\_train)}
\FunctionTok{write\_csv}\NormalTok{(amazon\_subset\_train, }\StringTok{"amazon\_review\_polarity\_csv/amazon\_subset\_train.csv"}\NormalTok{,}
    \AttributeTok{col\_names =} \ConstantTok{TRUE}\NormalTok{)}

\NormalTok{amazon\_train }\OtherTok{\textless{}{-}}\NormalTok{ amazon\_subset\_train}

\NormalTok{amazon\_train }\OtherTok{\textless{}{-}}\NormalTok{ amazon\_train }\SpecialCharTok{\%\textgreater{}\%}
    \FunctionTok{mutate}\NormalTok{(}\AttributeTok{label =} \FunctionTok{as.factor}\NormalTok{(label))}

\CommentTok{\# amazon\_val \textless{}{-} amazon\_train \%\textgreater{}\% mutate(label = as.factor(label))}
\end{Highlighting}
\end{Shaded}

\hypertarget{split-into-testtrain-and-create-resampling-folds}{%
\subsection{Split into test/train and create resampling
folds}\label{split-into-testtrain-and-create-resampling-folds}}

\begin{Shaded}
\begin{Highlighting}[]
\FunctionTok{set.seed}\NormalTok{(}\DecValTok{1234}\NormalTok{)}
\NormalTok{amazon\_split }\OtherTok{\textless{}{-}}\NormalTok{ amazon\_train }\SpecialCharTok{\%\textgreater{}\%}
    \FunctionTok{initial\_split}\NormalTok{()}
\NormalTok{amazon\_train }\OtherTok{\textless{}{-}} \FunctionTok{training}\NormalTok{(amazon\_split)}
\NormalTok{amazon\_test }\OtherTok{\textless{}{-}} \FunctionTok{testing}\NormalTok{(amazon\_split)}
\FunctionTok{set.seed}\NormalTok{(}\DecValTok{123}\NormalTok{)}
\NormalTok{amazon\_folds }\OtherTok{\textless{}{-}} \FunctionTok{vfold\_cv}\NormalTok{(amazon\_train)}
\CommentTok{\# amazon\_folds}
\end{Highlighting}
\end{Shaded}

\hypertarget{recipe-for-data-preprocessing}{%
\subsection{Recipe for data
preprocessing}\label{recipe-for-data-preprocessing}}

``step\_tfidf'' creates a specification of a recipe step that will
convert a tokenlist into multiple variables containing the
\href{https://www.tidytextmining.com/tfidf.html}{term frequency-inverse
document frequency} of tokens.(check it out in the console by typing
?textrecipes::step\_tfidf)

\begin{Shaded}
\begin{Highlighting}[]
\CommentTok{\# library(textrecipes)}

\NormalTok{amazon\_rec }\OtherTok{\textless{}{-}} \FunctionTok{recipe}\NormalTok{(label }\SpecialCharTok{\textasciitilde{}}\NormalTok{ text, }\AttributeTok{data =}\NormalTok{ amazon\_train) }\SpecialCharTok{\%\textgreater{}\%}
    \FunctionTok{step\_tokenize}\NormalTok{(text) }\SpecialCharTok{\%\textgreater{}\%}
    \FunctionTok{step\_tokenfilter}\NormalTok{(text, }\AttributeTok{max\_tokens =} \DecValTok{5000}\NormalTok{) }\SpecialCharTok{\%\textgreater{}\%}
    \FunctionTok{step\_tfidf}\NormalTok{(text)}

\NormalTok{amazon\_rec}
\NormalTok{Data Recipe}

\NormalTok{Inputs}\SpecialCharTok{:}

\NormalTok{      role }\CommentTok{\#variables}
\NormalTok{   outcome          }\DecValTok{1}
\NormalTok{ predictor          }\DecValTok{1}

\NormalTok{Operations}\SpecialCharTok{:}

\NormalTok{Tokenization }\ControlFlowTok{for}\NormalTok{ text}
\NormalTok{Text filtering }\ControlFlowTok{for}\NormalTok{ text}
\NormalTok{Term frequency}\SpecialCharTok{{-}}\NormalTok{inverse document frequency with text}
\end{Highlighting}
\end{Shaded}

\hypertarget{lasso-regularized-classification-model-and-tuning}{%
\subsection{Lasso regularized classification model and
tuning}\label{lasso-regularized-classification-model-and-tuning}}

Linear models are not considered cutting edge in NLP research, but are a
workhorse in real-world practice. Here we will use a lasso regularized
model \href{http://www.jstor.org/stable/2346178}{Tibshirani, Robert.
1996. ``Regression Shrinkage and Selection via the Lasso.'' Journal of
the Royal Statistical Society. Series B (Methodological) 58 (1). Royal
Statistical Society, Wiley: 267--288.}.

Let's create a specification of lasso regularized model.

``penalty'' is a model hyperparameter and we cannot learn its best value
during model training, but we can estimate the best value by training
many models on resampled data sets and exploring how well all these
models perform. Let's build a new model specification for model tuning.

\begin{Shaded}
\begin{Highlighting}[]
\NormalTok{lasso\_spec }\OtherTok{\textless{}{-}} \FunctionTok{logistic\_reg}\NormalTok{(}\AttributeTok{penalty =} \FunctionTok{tune}\NormalTok{(), }\AttributeTok{mixture =} \DecValTok{1}\NormalTok{) }\SpecialCharTok{\%\textgreater{}\%}
    \FunctionTok{set\_mode}\NormalTok{(}\StringTok{"classification"}\NormalTok{) }\SpecialCharTok{\%\textgreater{}\%}
    \FunctionTok{set\_engine}\NormalTok{(}\StringTok{"glmnet"}\NormalTok{)}

\NormalTok{lasso\_spec}
\NormalTok{Logistic Regression Model }\FunctionTok{Specification}\NormalTok{ (classification)}

\NormalTok{Main Arguments}\SpecialCharTok{:}
\NormalTok{  penalty }\OtherTok{=} \FunctionTok{tune}\NormalTok{()}
\NormalTok{  mixture }\OtherTok{=} \DecValTok{1}

\NormalTok{Computational engine}\SpecialCharTok{:}\NormalTok{ glmnet }
\end{Highlighting}
\end{Shaded}

\hypertarget{a-model-workflow}{%
\subsection{A model workflow}\label{a-model-workflow}}

We need a few more components before we can tune our workflow. Let's use
a sparse data encoding.

We can change how our text data is represented to take advantage of its
sparsity, especially for models like lasso regularized models. The
regularized regression model we trained above used
set\_engine(``glmnet''); this computational engine can be more efficient
when text data is transformed to a sparse matrix, rather than a dense
data frame or tibble representation.

To keep our text data sparse throughout modeling and use the sparse
capabilities of set\_engine(``glmnet''), we need to explicitly set a
non-default preprocessing blueprint, using the package hardhat
\href{https://CRAN.R-project.org/package=hardhat}{Vaughan, Davis, and
Max Kuhn. 2020. hardhat: Construct Modeling Packages.}.

\begin{Shaded}
\begin{Highlighting}[]
\FunctionTok{library}\NormalTok{(hardhat)}
\NormalTok{sparse\_bp }\OtherTok{\textless{}{-}} \FunctionTok{default\_recipe\_blueprint}\NormalTok{(}\AttributeTok{composition =} \StringTok{"dgCMatrix"}\NormalTok{)}
\end{Highlighting}
\end{Shaded}

Let's create a grid of possible regularization penalties to try, using a
convenience function for penalty() called grid\_regular() from the dials
package.

\begin{Shaded}
\begin{Highlighting}[]
\NormalTok{lambda\_grid }\OtherTok{\textless{}{-}} \FunctionTok{grid\_regular}\NormalTok{(}\FunctionTok{penalty}\NormalTok{(}\AttributeTok{range =} \FunctionTok{c}\NormalTok{(}\SpecialCharTok{{-}}\DecValTok{5}\NormalTok{, }\DecValTok{0}\NormalTok{)), }\AttributeTok{levels =} \DecValTok{20}\NormalTok{)}
\NormalTok{lambda\_grid}
\end{Highlighting}
\end{Shaded}

\begin{tabular}{r}
\hline
penalty\\
\hline
0.0000100\\
\hline
0.0000183\\
\hline
0.0000336\\
\hline
0.0000616\\
\hline
0.0001129\\
\hline
0.0002069\\
\hline
0.0003793\\
\hline
0.0006952\\
\hline
0.0012743\\
\hline
0.0023357\\
\hline
0.0042813\\
\hline
0.0078476\\
\hline
0.0143845\\
\hline
0.0263665\\
\hline
0.0483293\\
\hline
0.0885867\\
\hline
0.1623777\\
\hline
0.2976351\\
\hline
0.5455595\\
\hline
1.0000000\\
\hline
\end{tabular}

Now these can be combined in a tuneable workflow()

\begin{Shaded}
\begin{Highlighting}[]
\NormalTok{amazon\_wf }\OtherTok{\textless{}{-}} \FunctionTok{workflow}\NormalTok{() }\SpecialCharTok{\%\textgreater{}\%}
    \FunctionTok{add\_recipe}\NormalTok{(amazon\_rec, }\AttributeTok{blueprint =}\NormalTok{ sparse\_bp) }\SpecialCharTok{\%\textgreater{}\%}
    \FunctionTok{add\_model}\NormalTok{(lasso\_spec)}

\NormalTok{amazon\_wf}
\SpecialCharTok{==}\NormalTok{ Workflow }\SpecialCharTok{==}\ErrorTok{==================================================================}
\NormalTok{Preprocessor}\SpecialCharTok{:}\NormalTok{ Recipe}
\NormalTok{Model}\SpecialCharTok{:} \FunctionTok{logistic\_reg}\NormalTok{()}

\SpecialCharTok{{-}{-}}\NormalTok{ Preprocessor }\SpecialCharTok{{-}{-}{-}{-}{-}{-}{-}{-}{-}{-}{-}{-}{-}{-}{-}{-}{-}{-}{-}{-}{-}{-}{-}{-}{-}{-}{-}{-}{-}{-}{-}{-}{-}{-}{-}{-}{-}{-}{-}{-}{-}{-}{-}{-}{-}{-}{-}{-}{-}{-}{-}{-}{-}{-}{-}{-}{-}{-}{-}{-}{-}{-}{-}{-}}
\DecValTok{3}\NormalTok{ Recipe Steps}

\SpecialCharTok{*} \FunctionTok{step\_tokenize}\NormalTok{()}
\SpecialCharTok{*} \FunctionTok{step\_tokenfilter}\NormalTok{()}
\SpecialCharTok{*} \FunctionTok{step\_tfidf}\NormalTok{()}

\SpecialCharTok{{-}{-}}\NormalTok{ Model }\SpecialCharTok{{-}{-}{-}{-}{-}{-}{-}{-}{-}{-}{-}{-}{-}{-}{-}{-}{-}{-}{-}{-}{-}{-}{-}{-}{-}{-}{-}{-}{-}{-}{-}{-}{-}{-}{-}{-}{-}{-}{-}{-}{-}{-}{-}{-}{-}{-}{-}{-}{-}{-}{-}{-}{-}{-}{-}{-}{-}{-}{-}{-}{-}{-}{-}{-}{-}{-}{-}{-}{-}{-}{-}}
\NormalTok{Logistic Regression Model }\FunctionTok{Specification}\NormalTok{ (classification)}

\NormalTok{Main Arguments}\SpecialCharTok{:}
\NormalTok{  penalty }\OtherTok{=} \FunctionTok{tune}\NormalTok{()}
\NormalTok{  mixture }\OtherTok{=} \DecValTok{1}

\NormalTok{Computational engine}\SpecialCharTok{:}\NormalTok{ glmnet }
\end{Highlighting}
\end{Shaded}

\newpage

\hypertarget{tune-the-workflow}{%
\subsection{Tune the workflow}\label{tune-the-workflow}}

Let's use tune\_grid() to fit a model at each of the values for the
regularization penalty in our regular grid and every resample in
amazon\_folds.

\begin{Shaded}
\begin{Highlighting}[]
\FunctionTok{set.seed}\NormalTok{(}\DecValTok{2020}\NormalTok{)}
\NormalTok{lasso\_rs }\OtherTok{\textless{}{-}} \FunctionTok{tune\_grid}\NormalTok{(amazon\_wf, amazon\_folds, }\AttributeTok{grid =}\NormalTok{ lambda\_grid, }\AttributeTok{control =} \FunctionTok{control\_resamples}\NormalTok{(}\AttributeTok{save\_pred =} \ConstantTok{TRUE}\NormalTok{))}

\CommentTok{\# lasso\_rs}
\end{Highlighting}
\end{Shaded}

We now have a set of metrics for each value of the regularization
penalty.

We can extract the relevant information using collect\_metrics() and
collect\_predictions()

See Table \ref{tbl:lasso_metrics} for Lasso Metrics

\begin{Shaded}
\begin{Highlighting}[]
\NormalTok{m\_lm }\OtherTok{\textless{}{-}} \FunctionTok{collect\_metrics}\NormalTok{(lasso\_rs)}
\FunctionTok{kable}\NormalTok{(m\_lm, }\AttributeTok{format =} \StringTok{"simple"}\NormalTok{, }\AttributeTok{caption =} \StringTok{"Lasso Metrics}\SpecialCharTok{\textbackslash{}\textbackslash{}}\StringTok{label\{tbl:lasso\_metrics\}"}\NormalTok{)}
\end{Highlighting}
\end{Shaded}

\begin{longtable}[]{@{}rllrrrl@{}}
\caption{Lasso Metrics\label{tbl:lasso_metrics}}\tabularnewline
\toprule
penalty & .metric & .estimator & mean & n & std\_err & .config \\
\midrule
\endfirsthead
\toprule
penalty & .metric & .estimator & mean & n & std\_err & .config \\
\midrule
\endhead
0.0000100 & accuracy & binary & 0.8893912 & 10 & 0.0005032 &
Preprocessor1\_Model01 \\
0.0000100 & roc\_auc & binary & 0.9538016 & 10 & 0.0003006 &
Preprocessor1\_Model01 \\
0.0000183 & accuracy & binary & 0.8893912 & 10 & 0.0005032 &
Preprocessor1\_Model02 \\
0.0000183 & roc\_auc & binary & 0.9538016 & 10 & 0.0003006 &
Preprocessor1\_Model02 \\
0.0000336 & accuracy & binary & 0.8893912 & 10 & 0.0004969 &
Preprocessor1\_Model03 \\
0.0000336 & roc\_auc & binary & 0.9538247 & 10 & 0.0003008 &
Preprocessor1\_Model03 \\
0.0000616 & accuracy & binary & 0.8895062 & 10 & 0.0004782 &
Preprocessor1\_Model04 \\
0.0000616 & roc\_auc & binary & 0.9539107 & 10 & 0.0003016 &
Preprocessor1\_Model04 \\
0.0001129 & accuracy & binary & 0.8896719 & 10 & 0.0004941 &
Preprocessor1\_Model05 \\
0.0001129 & roc\_auc & binary & 0.9540329 & 10 & 0.0003014 &
Preprocessor1\_Model05 \\
0.0002069 & accuracy & binary & 0.8897064 & 10 & 0.0004424 &
Preprocessor1\_Model06 \\
0.0002069 & roc\_auc & binary & 0.9541505 & 10 & 0.0003024 &
Preprocessor1\_Model06 \\
0.0003793 & accuracy & binary & 0.8894809 & 10 & 0.0004733 &
Preprocessor1\_Model07 \\
0.0003793 & roc\_auc & binary & 0.9541045 & 10 & 0.0003048 &
Preprocessor1\_Model07 \\
0.0006952 & accuracy & binary & 0.8883996 & 10 & 0.0004462 &
Preprocessor1\_Model08 \\
0.0006952 & roc\_auc & binary & 0.9534221 & 10 & 0.0003069 &
Preprocessor1\_Model08 \\
0.0012743 & accuracy & binary & 0.8853696 & 10 & 0.0005228 &
Preprocessor1\_Model09 \\
0.0012743 & roc\_auc & binary & 0.9512859 & 10 & 0.0003173 &
Preprocessor1\_Model09 \\
0.0023357 & accuracy & binary & 0.8782767 & 10 & 0.0005656 &
Preprocessor1\_Model10 \\
0.0023357 & roc\_auc & binary & 0.9465029 & 10 & 0.0003413 &
Preprocessor1\_Model10 \\
0.0042813 & accuracy & binary & 0.8654758 & 10 & 0.0005853 &
Preprocessor1\_Model11 \\
0.0042813 & roc\_auc & binary & 0.9375896 & 10 & 0.0003789 &
Preprocessor1\_Model11 \\
0.0078476 & accuracy & binary & 0.8447308 & 10 & 0.0006268 &
Preprocessor1\_Model12 \\
0.0078476 & roc\_auc & binary & 0.9225346 & 10 & 0.0003517 &
Preprocessor1\_Model12 \\
0.0143845 & accuracy & binary & 0.8114196 & 10 & 0.0008523 &
Preprocessor1\_Model13 \\
0.0143845 & roc\_auc & binary & 0.8982350 & 10 & 0.0004307 &
Preprocessor1\_Model13 \\
0.0263665 & accuracy & binary & 0.7658711 & 10 & 0.0008243 &
Preprocessor1\_Model14 \\
0.0263665 & roc\_auc & binary & 0.8620465 & 10 & 0.0005772 &
Preprocessor1\_Model14 \\
0.0483293 & accuracy & binary & 0.7075908 & 10 & 0.0008798 &
Preprocessor1\_Model15 \\
0.0483293 & roc\_auc & binary & 0.8020162 & 10 & 0.0008407 &
Preprocessor1\_Model15 \\
0.0885867 & accuracy & binary & 0.6665424 & 10 & 0.0010263 &
Preprocessor1\_Model16 \\
0.0885867 & roc\_auc & binary & 0.7273321 & 10 & 0.0006962 &
Preprocessor1\_Model16 \\
0.1623777 & accuracy & binary & 0.5796028 & 10 & 0.0007071 &
Preprocessor1\_Model17 \\
0.1623777 & roc\_auc & binary & 0.5000000 & 10 & 0.0000000 &
Preprocessor1\_Model17 \\
0.2976351 & accuracy & binary & 0.5796028 & 10 & 0.0007071 &
Preprocessor1\_Model18 \\
0.2976351 & roc\_auc & binary & 0.5000000 & 10 & 0.0000000 &
Preprocessor1\_Model18 \\
0.5455595 & accuracy & binary & 0.5796028 & 10 & 0.0007071 &
Preprocessor1\_Model19 \\
0.5455595 & roc\_auc & binary & 0.5000000 & 10 & 0.0000000 &
Preprocessor1\_Model19 \\
1.0000000 & accuracy & binary & 0.5796028 & 10 & 0.0007071 &
Preprocessor1\_Model20 \\
1.0000000 & roc\_auc & binary & 0.5000000 & 10 & 0.0000000 &
Preprocessor1\_Model20 \\
\bottomrule
\end{longtable}

What are the best models?

See Table \ref{tbl:best_lasso_roc} for Best Lasso ROC.

\begin{Shaded}
\begin{Highlighting}[]
\NormalTok{m\_blr }\OtherTok{\textless{}{-}} \FunctionTok{show\_best}\NormalTok{(lasso\_rs, }\StringTok{"roc\_auc"}\NormalTok{)}
\FunctionTok{kable}\NormalTok{(m\_blr, }\AttributeTok{format =} \StringTok{"simple"}\NormalTok{, }\AttributeTok{caption =} \StringTok{"Best Lasso ROC}\SpecialCharTok{\textbackslash{}\textbackslash{}}\StringTok{label\{tbl:best\_lasso\_roc\}"}\NormalTok{)}
\end{Highlighting}
\end{Shaded}

\begin{longtable}[]{@{}rllrrrl@{}}
\caption{Best Lasso ROC\label{tbl:best_lasso_roc}}\tabularnewline
\toprule
penalty & .metric & .estimator & mean & n & std\_err & .config \\
\midrule
\endfirsthead
\toprule
penalty & .metric & .estimator & mean & n & std\_err & .config \\
\midrule
\endhead
0.0002069 & roc\_auc & binary & 0.9541505 & 10 & 0.0003024 &
Preprocessor1\_Model06 \\
0.0003793 & roc\_auc & binary & 0.9541045 & 10 & 0.0003048 &
Preprocessor1\_Model07 \\
0.0001129 & roc\_auc & binary & 0.9540329 & 10 & 0.0003014 &
Preprocessor1\_Model05 \\
0.0000616 & roc\_auc & binary & 0.9539107 & 10 & 0.0003016 &
Preprocessor1\_Model04 \\
0.0000336 & roc\_auc & binary & 0.9538247 & 10 & 0.0003008 &
Preprocessor1\_Model03 \\
\bottomrule
\end{longtable}

See Table \ref{tbl:best_lasso_acc} for Best Lasso Accuracy.

\begin{Shaded}
\begin{Highlighting}[]
\NormalTok{m\_bla }\OtherTok{\textless{}{-}} \FunctionTok{show\_best}\NormalTok{(lasso\_rs, }\StringTok{"accuracy"}\NormalTok{)}
\FunctionTok{kable}\NormalTok{(m\_bla, }\AttributeTok{format =} \StringTok{"simple"}\NormalTok{, }\AttributeTok{caption =} \StringTok{"Best Lasso Accuracy}\SpecialCharTok{\textbackslash{}\textbackslash{}}\StringTok{label\{tbl:best\_lasso\_acc\}"}\NormalTok{)}
\end{Highlighting}
\end{Shaded}

\begin{longtable}[]{@{}rllrrrl@{}}
\caption{Best Lasso Accuracy\label{tbl:best_lasso_acc}}\tabularnewline
\toprule
penalty & .metric & .estimator & mean & n & std\_err & .config \\
\midrule
\endfirsthead
\toprule
penalty & .metric & .estimator & mean & n & std\_err & .config \\
\midrule
\endhead
0.0002069 & accuracy & binary & 0.8897064 & 10 & 0.0004424 &
Preprocessor1\_Model06 \\
0.0001129 & accuracy & binary & 0.8896719 & 10 & 0.0004941 &
Preprocessor1\_Model05 \\
0.0000616 & accuracy & binary & 0.8895062 & 10 & 0.0004782 &
Preprocessor1\_Model04 \\
0.0003793 & accuracy & binary & 0.8894809 & 10 & 0.0004733 &
Preprocessor1\_Model07 \\
0.0000100 & accuracy & binary & 0.8893912 & 10 & 0.0005032 &
Preprocessor1\_Model01 \\
\bottomrule
\end{longtable}

Let's visualize these metrics; accuracy and ROC AUC, in Figure
\ref{fig:model_7} to see what the best model is.

\begin{figure}
\centering
\includegraphics{figures/plot_lasso-1.pdf}
\caption{Lasso model performance across regularization
penalties\label{fig:model_7}}
\end{figure}

See Table \ref{tbl:lasso_predictions} for Lasso Predictions

\begin{Shaded}
\begin{Highlighting}[]
\NormalTok{m\_lp }\OtherTok{\textless{}{-}} \FunctionTok{collect\_predictions}\NormalTok{(lasso\_rs)}
\FunctionTok{kable}\NormalTok{(}\FunctionTok{head}\NormalTok{(m\_lp), }\AttributeTok{format =} \StringTok{"simple"}\NormalTok{, }\AttributeTok{caption =} \StringTok{"Lasso Predictions}\SpecialCharTok{\textbackslash{}\textbackslash{}}\StringTok{label\{tbl:lasso\_predictions\}"}\NormalTok{)}
\end{Highlighting}
\end{Shaded}

\begin{longtable}[]{@{}lrrrrlll@{}}
\caption{Lasso Predictions\label{tbl:lasso_predictions}}\tabularnewline
\toprule
id & .pred\_0 & .pred\_1 & .row & penalty & .pred\_class & label &
.config \\
\midrule
\endfirsthead
\toprule
id & .pred\_0 & .pred\_1 & .row & penalty & .pred\_class & label &
.config \\
\midrule
\endhead
Fold01 & 0.4909890 & 0.5090110 & 10 & 1e-05 & 1 & 0 &
Preprocessor1\_Model01 \\
Fold01 & 0.0116122 & 0.9883878 & 28 & 1e-05 & 1 & 1 &
Preprocessor1\_Model01 \\
Fold01 & 0.0145242 & 0.9854758 & 30 & 1e-05 & 1 & 1 &
Preprocessor1\_Model01 \\
Fold01 & 0.0144831 & 0.9855169 & 68 & 1e-05 & 1 & 1 &
Preprocessor1\_Model01 \\
Fold01 & 0.2691942 & 0.7308058 & 79 & 1e-05 & 1 & 0 &
Preprocessor1\_Model01 \\
Fold01 & 0.9884782 & 0.0115218 & 86 & 1e-05 & 0 & 0 &
Preprocessor1\_Model01 \\
\bottomrule
\end{longtable}

Figure \ref{fig:model_8} shows the ROC curve, a visualization of how
well a classification model can distinguish between classes

\begin{figure}
\centering
\includegraphics{figures/m_lp_roc_0-1.pdf}
\caption{Lasso model ROC Label 0\label{fig:model_8}}
\end{figure}

Figure \ref{fig:model_9} shows the ROC curve, a visualization of how
well a classification model can distinguish between classes

\begin{figure}
\centering
\includegraphics{figures/m_lp_roc_1-1.pdf}
\caption{Lasso model ROC Label 1\label{fig:model_9}}
\end{figure}

\newpage

\hypertarget{results}{%
\subsection{Results}\label{results}}

We saw that regularized linear models, such as lasso, often work well
for text data sets.

The default performance parameters for binary classification are
accuracy and ROC AUC (area under the receiver operator characteristic
curve). Here, the best accuracy is:

Best ROC\_AUC is 0.9541505\\
Best Accuracy is 0.8897064

As we go along, we will be comparing different approaches. Let's start
by creating a results table with this BLM to get Table
\ref{tbl:blm_results_table}:

\begin{longtable}[]{@{}llrl@{}}
\caption{Baseline Linear Model
Results\label{tbl:blm_results_table}}\tabularnewline
\toprule
Index & Method & Accuracy & Loss \\
\midrule
\endfirsthead
\toprule
Index & Method & Accuracy & Loss \\
\midrule
\endhead
1 & BLM & 0.8897064 & NA \\
\bottomrule
\end{longtable}

Accuracy and ROC AUC are performance metrics used for classification
models. For both, values closer to 1 are better.

Accuracy is the proportion of the data that are predicted correctly. Be
aware that accuracy can be misleading in some situations, such as for
imbalanced data sets.

ROC AUC measures how well a classifier performs at different thresholds.
The ROC curve plots the true positive rate against the false positive
rate, and AUC closer to 1 indicates a better-performing model while AUC
closer to 0.5 indicates a model that does no better than random
guessing.

Figure \ref{fig:model_8} and Figure \ref{fig:model_9} show the ROC
curves, a visualization of how well our classification model can
distinguish between classes.

The area under each of these curves is the roc\_auc metric we have
computed. If the curve was close to the diagonal line, then the model's
predictions would be no better than random guessing.

One metric alone cannot give you a complete picture of how well your
classification model is performing. The confusion matrix is a good
starting point to get an overview of your model performance, as it
includes rich information.

Another way to evaluate our model is to evaluate the confusion matrix. A
confusion matrix tabulates a model's false positives and false negatives
for each class. The function conf\_mat\_resampled() computes a separate
confusion matrix for each resample and takes the average of the cell
counts. This allows us to visualize an overall confusion matrix rather
than needing to examine each resample individually.

\hypertarget{preprocessing-for-rest-of-the-models}{%
\section{Preprocessing for rest of the
models}\label{preprocessing-for-rest-of-the-models}}

Preprocessing for deep learning models is different than preprocessing
for most other text models. These neural networks model sequences, so we
have to choose the length of sequences we would like to include.
Sequences that are longer than this length are truncated (information is
thrown away) and those that are shorter than this length are padded with
zeroes (an empty, non-informative value) to get to the chosen sequence
length. This sequence length is a hyperparameter of the model and we
need to select this value such that we don't overshoot and introduce a
lot of padded zeroes which would make the model hard to train, or
undershoot and cut off too much informative text.

We will use the recipes and textrecipes packages for data preprocessing
and feature engineering.

The formula used to specify this recipe \textasciitilde{} text does not
have an outcome, because we are using recipes and textrecipes functions
on their own, outside of the rest of the tidymodels framework; we don't
need to know about the outcome here. This preprocessing recipe tokenizes
our text and filters to keep only the top 20,000 words and then it
transforms the tokenized text into a numeric format appropriate for
modeling, using step\_sequence\_onehot().

\begin{Shaded}
\begin{Highlighting}[]
\FunctionTok{rm}\NormalTok{(amazon\_folds, amazon\_rec, amazon\_split, amazon\_test, amazon\_train, amazon\_wf,}
\NormalTok{    lambda\_grid, lasso\_rs, lasso\_spec, sparse\_bp)}

\FunctionTok{gc}\NormalTok{()}
            \FunctionTok{used}\NormalTok{  (Mb) gc }\FunctionTok{trigger}\NormalTok{   (Mb)   max }\FunctionTok{used}\NormalTok{    (Mb)}
\NormalTok{Ncells   }\DecValTok{4147050} \FloatTok{221.5}   \DecValTok{12861945}  \FloatTok{687.0}   \DecValTok{20005140}  \FloatTok{1068.4}
\NormalTok{Vcells }\DecValTok{112719032} \FloatTok{860.0}  \DecValTok{867481767} \FloatTok{6618.4} \DecValTok{4028113514} \FloatTok{30732.1}

\NormalTok{amazon\_subset\_train }\OtherTok{\textless{}{-}}\NormalTok{ readr}\SpecialCharTok{::}\FunctionTok{read\_csv}\NormalTok{(}\StringTok{"amazon\_review\_polarity\_csv/amazon\_subset\_train.csv"}\NormalTok{)}

\NormalTok{amazon\_train }\OtherTok{\textless{}{-}}\NormalTok{ amazon\_subset\_train}

\NormalTok{max\_words }\OtherTok{\textless{}{-}} \DecValTok{20000}
\NormalTok{max\_length }\OtherTok{\textless{}{-}} \DecValTok{30}
\NormalTok{mystopwords }\OtherTok{\textless{}{-}} \FunctionTok{c}\NormalTok{(}\StringTok{"s"}\NormalTok{, }\StringTok{"t"}\NormalTok{, }\StringTok{"m"}\NormalTok{, }\StringTok{"ve"}\NormalTok{, }\StringTok{"re"}\NormalTok{, }\StringTok{"d"}\NormalTok{, }\StringTok{"ll"}\NormalTok{)}

\NormalTok{amazon\_rec }\OtherTok{\textless{}{-}} \FunctionTok{recipe}\NormalTok{(}\SpecialCharTok{\textasciitilde{}}\NormalTok{text, }\AttributeTok{data =}\NormalTok{ amazon\_subset\_train) }\SpecialCharTok{\%\textgreater{}\%}
    \FunctionTok{step\_text\_normalization}\NormalTok{(text) }\SpecialCharTok{\%\textgreater{}\%}
    \FunctionTok{step\_tokenize}\NormalTok{(text) }\SpecialCharTok{\%\textgreater{}\%}
    \FunctionTok{step\_stopwords}\NormalTok{(text, }\AttributeTok{stopword\_source =} \StringTok{"stopwords{-}iso"}\NormalTok{, }\AttributeTok{custom\_stopword\_source =}\NormalTok{ mystopwords) }\SpecialCharTok{\%\textgreater{}\%}
    \FunctionTok{step\_tokenfilter}\NormalTok{(text, }\AttributeTok{max\_tokens =}\NormalTok{ max\_words) }\SpecialCharTok{\%\textgreater{}\%}
    \FunctionTok{step\_sequence\_onehot}\NormalTok{(text, }\AttributeTok{sequence\_length =}\NormalTok{ max\_length)}

\NormalTok{amazon\_rec}
\NormalTok{Data Recipe}

\NormalTok{Inputs}\SpecialCharTok{:}

\NormalTok{      role }\CommentTok{\#variables}
\NormalTok{ predictor          }\DecValTok{1}

\NormalTok{Operations}\SpecialCharTok{:}

\NormalTok{text\_normalizationming }\ControlFlowTok{for}\NormalTok{ text}
\NormalTok{Tokenization }\ControlFlowTok{for}\NormalTok{ text}
\NormalTok{Stop word removal }\ControlFlowTok{for}\NormalTok{ text}
\NormalTok{Text filtering }\ControlFlowTok{for}\NormalTok{ text}
\NormalTok{Sequence }\DecValTok{1}\NormalTok{ hot encoding }\ControlFlowTok{for}\NormalTok{ text}
\end{Highlighting}
\end{Shaded}

The prep() function will compute or estimate statistics from the
training set; the output of prep() is a prepped recipe.

When we bake() a prepped recipe, we apply the preprocessing to the data
set. We can get out the training set that we started with by specifying
new\_data = NULL or apply it to another set via new\_data =
my\_other\_data\_set. The output of bake() is a data set like a tibble
or a matrix, depending on the composition argument.

Let's now prepare and apply our feature engineering recipe amazon\_rec
so we can use it in our deep learning model.

\begin{Shaded}
\begin{Highlighting}[]
\NormalTok{amazon\_prep }\OtherTok{\textless{}{-}} \FunctionTok{prep}\NormalTok{(amazon\_rec)}

\NormalTok{amazon\_subset\_train }\OtherTok{\textless{}{-}} \FunctionTok{bake}\NormalTok{(amazon\_prep, }\AttributeTok{new\_data =} \ConstantTok{NULL}\NormalTok{, }\AttributeTok{composition =} \StringTok{"matrix"}\NormalTok{)}
\FunctionTok{dim}\NormalTok{(amazon\_subset\_train)}
\NormalTok{[}\DecValTok{1}\NormalTok{] }\DecValTok{579545}     \DecValTok{30}
\end{Highlighting}
\end{Shaded}

The prep() function will compute or estimate statistics from the
training set; the output of prep() is a prepped recipe. The prepped
recipe can be tidied using tidy() to extract the vocabulary, represented
in the vocabulary and token columns.

\begin{Shaded}
\begin{Highlighting}[]
\NormalTok{amazon\_prep }\SpecialCharTok{\%\textgreater{}\%}
    \FunctionTok{tidy}\NormalTok{(}\DecValTok{5}\NormalTok{) }\SpecialCharTok{\%\textgreater{}\%}
    \FunctionTok{head}\NormalTok{(}\DecValTok{10}\NormalTok{)}
\end{Highlighting}
\end{Shaded}

\begin{tabular}{l|r|l|l}
\hline
terms & vocabulary & token & id\\
\hline
text & 1 & a & sequence\_onehot\_bCTRZ\\
\hline
text & 2 & à & sequence\_onehot\_bCTRZ\\
\hline
text & 3 & aa & sequence\_onehot\_bCTRZ\\
\hline
text & 4 & aaa & sequence\_onehot\_bCTRZ\\
\hline
text & 5 & aaaa & sequence\_onehot\_bCTRZ\\
\hline
text & 6 & aaliyah & sequence\_onehot\_bCTRZ\\
\hline
text & 7 & aaron & sequence\_onehot\_bCTRZ\\
\hline
text & 8 & ab & sequence\_onehot\_bCTRZ\\
\hline
text & 9 & abandon & sequence\_onehot\_bCTRZ\\
\hline
text & 10 & abandoned & sequence\_onehot\_bCTRZ\\
\hline
\end{tabular}

\newpage

\hypertarget{model-dnn}{%
\section{Model DNN}\label{model-dnn}}

A densely connected neural network is one of the simplest configurations
for a deep learning model and is typically not a model that will achieve
the highest performance on text data, but it is a good place to start to
understand the process of building and evaluating deep learning models
for text.

In a densely connected neural network, layers are fully connected
(dense) by the neurons in a network layer. Each neuron in a layer
receives an input from all the neurons present in the previous layer -
thus, they're densely connected.

The input comes in to the network all at once and is densely (in this
case, fully) connected to the first hidden layer. A layer is ``hidden''
in the sense that it doesn't connect to the outside world; the input and
output layers take care of this. The neurons in any given layer are only
connected to the next layer. The numbers of layers and nodes within each
layer are variable and are hyperparameters of the model selected by us.

\hypertarget{a-simple-flattened-dense-neural-network}{%
\subsection{A Simple flattened dense neural
network}\label{a-simple-flattened-dense-neural-network}}

Our first deep learning model embeds the Amazon Reviews in sequences of
vectors, flattens them, and then trains a dense network layer to predict
whether the Review was positive(1) or not(0).

\begin{enumerate}
\def\labelenumi{\arabic{enumi}.}
\item
  We initiate the Keras model as a linear stack of layers with
  keras\_model\_sequential().
\item
  Our first layer - layer\_embedding() turns each observation into an
  (embedding\_dim * sequence\_length) = 12 * 30
\item
  In total, we will create a (number\_of\_observations * embedding\_dim
  * sequence\_length) data cube.
\item
  The next layer\_flatten() layer takes the matrix for each observation
  and flattens them down into one dimension. This will create a (30 *
  12) = 360 long vector for each observation.
\item
  layer\_layer\_normalization() - Normalize the activations of the
  previous layer for each given example in a batch independently.
\item
  Lastly, we have 2 densely connected layers. The last layer has a
  sigmoid activation function to give us an output between 0 and 1,
  since we want to model a probability for a binary classification
  problem.
\end{enumerate}

\begin{Shaded}
\begin{Highlighting}[]
\CommentTok{\# library(keras) use\_python(python =}
\CommentTok{\# \textquotesingle{}/c/Users/bijoor/.conda/envs/tensorflow{-}python/python.exe\textquotesingle{}, required = TRUE)}
\CommentTok{\# use\_condaenv(condaenv = \textquotesingle{}tensorflow{-}python\textquotesingle{}, required = TRUE)}

\NormalTok{dense\_model }\OtherTok{\textless{}{-}} \FunctionTok{keras\_model\_sequential}\NormalTok{() }\SpecialCharTok{\%\textgreater{}\%}
    \FunctionTok{layer\_embedding}\NormalTok{(}\AttributeTok{input\_dim =}\NormalTok{ max\_words }\SpecialCharTok{+} \DecValTok{1}\NormalTok{, }\AttributeTok{output\_dim =} \DecValTok{12}\NormalTok{, }\AttributeTok{input\_length =}\NormalTok{ max\_length) }\SpecialCharTok{\%\textgreater{}\%}
    \FunctionTok{layer\_flatten}\NormalTok{() }\SpecialCharTok{\%\textgreater{}\%}
    \FunctionTok{layer\_layer\_normalization}\NormalTok{() }\SpecialCharTok{\%\textgreater{}\%}
    \CommentTok{\# layer\_dropout(0.1) \%\textgreater{}\%}
\FunctionTok{layer\_dense}\NormalTok{(}\AttributeTok{units =} \DecValTok{64}\NormalTok{) }\SpecialCharTok{\%\textgreater{}\%}
    \CommentTok{\# layer\_activation\_leaky\_relu() \%\textgreater{}\%}
\FunctionTok{layer\_activation\_relu}\NormalTok{() }\SpecialCharTok{\%\textgreater{}\%}
    \FunctionTok{layer\_dense}\NormalTok{(}\AttributeTok{units =} \DecValTok{1}\NormalTok{, }\AttributeTok{activation =} \StringTok{"sigmoid"}\NormalTok{)}

\NormalTok{dense\_model}
\NormalTok{Model}
\NormalTok{Model}\SpecialCharTok{:} \StringTok{"sequential"}
\NormalTok{\_\_\_\_\_\_\_\_\_\_\_\_\_\_\_\_\_\_\_\_\_\_\_\_\_\_\_\_\_\_\_\_\_\_\_\_\_\_\_\_\_\_\_\_\_\_\_\_\_\_\_\_\_\_\_\_\_\_\_\_\_\_\_\_\_\_\_\_\_\_\_\_\_\_\_\_\_\_\_\_}
\FunctionTok{Layer}\NormalTok{ (type)                        Output Shape                    Param }\CommentTok{\#     }
\SpecialCharTok{==}\ErrorTok{==============================================================================}
\FunctionTok{embedding}\NormalTok{ (Embedding)               (None, }\DecValTok{30}\NormalTok{, }\DecValTok{12}\NormalTok{)                  }\DecValTok{240012}      
\NormalTok{\_\_\_\_\_\_\_\_\_\_\_\_\_\_\_\_\_\_\_\_\_\_\_\_\_\_\_\_\_\_\_\_\_\_\_\_\_\_\_\_\_\_\_\_\_\_\_\_\_\_\_\_\_\_\_\_\_\_\_\_\_\_\_\_\_\_\_\_\_\_\_\_\_\_\_\_\_\_\_\_}
\FunctionTok{flatten}\NormalTok{ (Flatten)                   (None, }\DecValTok{360}\NormalTok{)                     }\DecValTok{0}           
\NormalTok{\_\_\_\_\_\_\_\_\_\_\_\_\_\_\_\_\_\_\_\_\_\_\_\_\_\_\_\_\_\_\_\_\_\_\_\_\_\_\_\_\_\_\_\_\_\_\_\_\_\_\_\_\_\_\_\_\_\_\_\_\_\_\_\_\_\_\_\_\_\_\_\_\_\_\_\_\_\_\_\_}
\FunctionTok{layer\_normalization}\NormalTok{ (}\FunctionTok{LayerNormaliza}\NormalTok{ (None, }\DecValTok{360}\NormalTok{)                     }\DecValTok{720}         
\NormalTok{\_\_\_\_\_\_\_\_\_\_\_\_\_\_\_\_\_\_\_\_\_\_\_\_\_\_\_\_\_\_\_\_\_\_\_\_\_\_\_\_\_\_\_\_\_\_\_\_\_\_\_\_\_\_\_\_\_\_\_\_\_\_\_\_\_\_\_\_\_\_\_\_\_\_\_\_\_\_\_\_}
\FunctionTok{dense\_1}\NormalTok{ (Dense)                     (None, }\DecValTok{64}\NormalTok{)                      }\DecValTok{23104}       
\NormalTok{\_\_\_\_\_\_\_\_\_\_\_\_\_\_\_\_\_\_\_\_\_\_\_\_\_\_\_\_\_\_\_\_\_\_\_\_\_\_\_\_\_\_\_\_\_\_\_\_\_\_\_\_\_\_\_\_\_\_\_\_\_\_\_\_\_\_\_\_\_\_\_\_\_\_\_\_\_\_\_\_}
\FunctionTok{re\_lu}\NormalTok{ (ReLU)                        (None, }\DecValTok{64}\NormalTok{)                      }\DecValTok{0}           
\NormalTok{\_\_\_\_\_\_\_\_\_\_\_\_\_\_\_\_\_\_\_\_\_\_\_\_\_\_\_\_\_\_\_\_\_\_\_\_\_\_\_\_\_\_\_\_\_\_\_\_\_\_\_\_\_\_\_\_\_\_\_\_\_\_\_\_\_\_\_\_\_\_\_\_\_\_\_\_\_\_\_\_}
\FunctionTok{dense}\NormalTok{ (Dense)                       (None, }\DecValTok{1}\NormalTok{)                       }\DecValTok{65}          
\SpecialCharTok{==}\ErrorTok{==============================================================================}
\NormalTok{Total params}\SpecialCharTok{:} \DecValTok{263}\NormalTok{,}\DecValTok{901}
\NormalTok{Trainable params}\SpecialCharTok{:} \DecValTok{263}\NormalTok{,}\DecValTok{901}
\NormalTok{Non}\SpecialCharTok{{-}}\NormalTok{trainable params}\SpecialCharTok{:} \DecValTok{0}
\NormalTok{\_\_\_\_\_\_\_\_\_\_\_\_\_\_\_\_\_\_\_\_\_\_\_\_\_\_\_\_\_\_\_\_\_\_\_\_\_\_\_\_\_\_\_\_\_\_\_\_\_\_\_\_\_\_\_\_\_\_\_\_\_\_\_\_\_\_\_\_\_\_\_\_\_\_\_\_\_\_\_\_}
\end{Highlighting}
\end{Shaded}

Before we can fit this model to the data it requires an optimizer and a
loss function to be able to compile.

When the neural network finishes passing a batch of data through the
network, it needs a way to use the difference between the predicted
values and true values to update the network's weights. The algorithm
that determines those weights is known as the optimization algorithm.
Many optimizers are available within Keras.

We will choose one of the following based on our previous
experimentation.

optimizer\_adam() - Adam - A Method for Stochastic Optimization

optimizer\_sgd() - Stochastic gradient descent optimizer

We can also use various options during training using the compile()
function, such as optimizer, loss and metrics.

\begin{Shaded}
\begin{Highlighting}[]
\CommentTok{\# opt \textless{}{-} optimizer\_adam(lr = 0.0001, decay = 1e{-}6) opt \textless{}{-} optimizer\_sgd(lr =}
\CommentTok{\# 0.001, decay = 1e{-}6)}
\NormalTok{opt }\OtherTok{\textless{}{-}} \FunctionTok{optimizer\_sgd}\NormalTok{()}
\NormalTok{dense\_model }\SpecialCharTok{\%\textgreater{}\%}
    \FunctionTok{compile}\NormalTok{(}\AttributeTok{optimizer =}\NormalTok{ opt, }\AttributeTok{loss =} \StringTok{"binary\_crossentropy"}\NormalTok{, }\AttributeTok{metrics =} \FunctionTok{c}\NormalTok{(}\StringTok{"accuracy"}\NormalTok{))}
\end{Highlighting}
\end{Shaded}

Finally, we can fit this model.

Here we specify the Keras defaults for creating a validation split and
tracking metrics with an internal validation split of 20\%.

\begin{Shaded}
\begin{Highlighting}[]
\NormalTok{dense\_history }\OtherTok{\textless{}{-}}\NormalTok{ dense\_model }\SpecialCharTok{\%\textgreater{}\%}
    \FunctionTok{fit}\NormalTok{(}\AttributeTok{x =}\NormalTok{ amazon\_subset\_train, }\AttributeTok{y =}\NormalTok{ amazon\_train}\SpecialCharTok{$}\NormalTok{label, }\AttributeTok{batch\_size =} \DecValTok{1024}\NormalTok{, }\AttributeTok{epochs =} \DecValTok{50}\NormalTok{,}
        \AttributeTok{initial\_epoch =} \DecValTok{0}\NormalTok{, }\AttributeTok{validation\_split =} \FloatTok{0.2}\NormalTok{, }\AttributeTok{verbose =} \DecValTok{2}\NormalTok{)}

\NormalTok{dense\_history}

\NormalTok{Final }\FunctionTok{epoch}\NormalTok{ (plot to see history)}\SpecialCharTok{:}
\NormalTok{        loss}\SpecialCharTok{:} \FloatTok{0.2457}
\NormalTok{    accuracy}\SpecialCharTok{:} \FloatTok{0.8994}
\NormalTok{    val\_loss}\SpecialCharTok{:} \FloatTok{0.2688}
\NormalTok{val\_accuracy}\SpecialCharTok{:} \FloatTok{0.8898} 
\end{Highlighting}
\end{Shaded}

``DNN Model 1 Fit History using validation\_split'' Figure
\ref{fig:model_10}

\begin{figure}
\centering
\includegraphics{figures/dense_model_1_split_hist-1.pdf}
\caption{DNN Model 1 Fit History using
validation\_split\label{fig:model_10}}
\end{figure}

\newpage

\hypertarget{evaluation}{%
\subsection{Evaluation}\label{evaluation}}

Instead of using Keras defaults, we can use tidymodels functions to be
more specific about these model characteristics. Instead of using the
validation\_split argument to fit(), we can create our own validation
set using tidymodels and use validation\_data argument for fit(). We
create our validation split from the training set.

\begin{Shaded}
\begin{Highlighting}[]
\FunctionTok{set.seed}\NormalTok{(}\DecValTok{234}\NormalTok{)}
\NormalTok{amazon\_val\_eval }\OtherTok{\textless{}{-}} \FunctionTok{validation\_split}\NormalTok{(amazon\_train, }\AttributeTok{strata =}\NormalTok{ label)}
\CommentTok{\# amazon\_val\_eval \textless{}\textless{} I am getting a pandoc stack error printing this}
\end{Highlighting}
\end{Shaded}

The split object contains the information necessary to extract the data
we will use for training/analysis and the data we will use for
validation/assessment. We can extract these data sets in their raw,
unprocessed form from the split using the helper functions analysis()
and assessment(). Then, we can apply our prepped preprocessing recipe
amazon\_prep to both to transform this data to the appropriate format
for our neural network architecture.

\begin{Shaded}
\begin{Highlighting}[]
\NormalTok{amazon\_analysis }\OtherTok{\textless{}{-}} \FunctionTok{bake}\NormalTok{(amazon\_prep, }\AttributeTok{new\_data =} \FunctionTok{analysis}\NormalTok{(amazon\_val\_eval}\SpecialCharTok{$}\NormalTok{splits[[}\DecValTok{1}\NormalTok{]]),}
    \AttributeTok{composition =} \StringTok{"matrix"}\NormalTok{)}
\FunctionTok{dim}\NormalTok{(amazon\_analysis)}
\NormalTok{[}\DecValTok{1}\NormalTok{] }\DecValTok{434658}     \DecValTok{30}
\end{Highlighting}
\end{Shaded}

\begin{Shaded}
\begin{Highlighting}[]
\NormalTok{amazon\_assess }\OtherTok{\textless{}{-}} \FunctionTok{bake}\NormalTok{(amazon\_prep, }\AttributeTok{new\_data =} \FunctionTok{assessment}\NormalTok{(amazon\_val\_eval}\SpecialCharTok{$}\NormalTok{splits[[}\DecValTok{1}\NormalTok{]]),}
    \AttributeTok{composition =} \StringTok{"matrix"}\NormalTok{)}
\FunctionTok{dim}\NormalTok{(amazon\_assess)}
\NormalTok{[}\DecValTok{1}\NormalTok{] }\DecValTok{144887}     \DecValTok{30}
\end{Highlighting}
\end{Shaded}

Here we get outcome variables for both sets.

\begin{Shaded}
\begin{Highlighting}[]
\NormalTok{label\_analysis }\OtherTok{\textless{}{-}} \FunctionTok{analysis}\NormalTok{(amazon\_val\_eval}\SpecialCharTok{$}\NormalTok{splits[[}\DecValTok{1}\NormalTok{]]) }\SpecialCharTok{\%\textgreater{}\%}
    \FunctionTok{pull}\NormalTok{(label)}
\NormalTok{label\_assess }\OtherTok{\textless{}{-}} \FunctionTok{assessment}\NormalTok{(amazon\_val\_eval}\SpecialCharTok{$}\NormalTok{splits[[}\DecValTok{1}\NormalTok{]]) }\SpecialCharTok{\%\textgreater{}\%}
    \FunctionTok{pull}\NormalTok{(label)}
\end{Highlighting}
\end{Shaded}

Let's setup a new DNN model 2.

Here we use layer\_dropout() - Dropout consists in randomly setting a
fraction rate of input units to 0 at each update during training time,
which helps prevent overfitting.

\begin{Shaded}
\begin{Highlighting}[]
\NormalTok{dense\_model }\OtherTok{\textless{}{-}} \FunctionTok{keras\_model\_sequential}\NormalTok{() }\SpecialCharTok{\%\textgreater{}\%}
    \FunctionTok{layer\_embedding}\NormalTok{(}\AttributeTok{input\_dim =}\NormalTok{ max\_words }\SpecialCharTok{+} \DecValTok{1}\NormalTok{, }\AttributeTok{output\_dim =} \DecValTok{12}\NormalTok{, }\AttributeTok{input\_length =}\NormalTok{ max\_length) }\SpecialCharTok{\%\textgreater{}\%}
    \FunctionTok{layer\_flatten}\NormalTok{() }\SpecialCharTok{\%\textgreater{}\%}
    \FunctionTok{layer\_layer\_normalization}\NormalTok{() }\SpecialCharTok{\%\textgreater{}\%}
    \FunctionTok{layer\_dropout}\NormalTok{(}\FloatTok{0.5}\NormalTok{) }\SpecialCharTok{\%\textgreater{}\%}
    \FunctionTok{layer\_dense}\NormalTok{(}\AttributeTok{units =} \DecValTok{64}\NormalTok{) }\SpecialCharTok{\%\textgreater{}\%}
    \FunctionTok{layer\_activation\_relu}\NormalTok{() }\SpecialCharTok{\%\textgreater{}\%}
    \FunctionTok{layer\_dropout}\NormalTok{(}\FloatTok{0.5}\NormalTok{) }\SpecialCharTok{\%\textgreater{}\%}
    \FunctionTok{layer\_dense}\NormalTok{(}\AttributeTok{units =} \DecValTok{128}\NormalTok{) }\SpecialCharTok{\%\textgreater{}\%}
    \FunctionTok{layer\_activation\_relu}\NormalTok{() }\SpecialCharTok{\%\textgreater{}\%}
    \FunctionTok{layer\_dense}\NormalTok{(}\AttributeTok{units =} \DecValTok{128}\NormalTok{) }\SpecialCharTok{\%\textgreater{}\%}
    \FunctionTok{layer\_activation\_relu}\NormalTok{() }\SpecialCharTok{\%\textgreater{}\%}
    \FunctionTok{layer\_dense}\NormalTok{(}\AttributeTok{units =} \DecValTok{1}\NormalTok{, }\AttributeTok{activation =} \StringTok{"sigmoid"}\NormalTok{)}

\NormalTok{opt }\OtherTok{\textless{}{-}} \FunctionTok{optimizer\_adam}\NormalTok{(}\AttributeTok{lr =} \FloatTok{1e{-}04}\NormalTok{, }\AttributeTok{decay =} \FloatTok{1e{-}06}\NormalTok{)}
\CommentTok{\# opt \textless{}{-} optimizer\_sgd(lr = 0.001, decay = 1e{-}6) opt \textless{}{-} optimizer\_sgd()}
\NormalTok{dense\_model }\SpecialCharTok{\%\textgreater{}\%}
    \FunctionTok{compile}\NormalTok{(}\AttributeTok{optimizer =}\NormalTok{ opt, }\AttributeTok{loss =} \StringTok{"binary\_crossentropy"}\NormalTok{, }\AttributeTok{metrics =} \FunctionTok{c}\NormalTok{(}\StringTok{"accuracy"}\NormalTok{))}
\end{Highlighting}
\end{Shaded}

We now fit this model to validation\_data - amazon\_assess and
label\_assess instead of the Keras default validation\_split.

\begin{Shaded}
\begin{Highlighting}[]
\NormalTok{val\_history }\OtherTok{\textless{}{-}}\NormalTok{ dense\_model }\SpecialCharTok{\%\textgreater{}\%}
    \FunctionTok{fit}\NormalTok{(}\AttributeTok{x =}\NormalTok{ amazon\_analysis, }\AttributeTok{y =}\NormalTok{ label\_analysis, }\AttributeTok{batch\_size =} \DecValTok{2048}\NormalTok{, }\AttributeTok{epochs =} \DecValTok{20}\NormalTok{,}
        \AttributeTok{validation\_data =} \FunctionTok{list}\NormalTok{(amazon\_assess, label\_assess), }\AttributeTok{verbose =} \DecValTok{2}\NormalTok{)}

\NormalTok{val\_history}

\NormalTok{Final }\FunctionTok{epoch}\NormalTok{ (plot to see history)}\SpecialCharTok{:}
\NormalTok{        loss}\SpecialCharTok{:} \FloatTok{0.2638}
\NormalTok{    accuracy}\SpecialCharTok{:} \FloatTok{0.8932}
\NormalTok{    val\_loss}\SpecialCharTok{:} \FloatTok{0.2837}
\NormalTok{val\_accuracy}\SpecialCharTok{:} \FloatTok{0.8961} 
\end{Highlighting}
\end{Shaded}

``DNN Model 2 Fit History using validation\_data'' Figure
\ref{fig:model_11}

\begin{figure}
\centering
\includegraphics{figures/dense_model_2_fit_val_d_hist-1.pdf}
\caption{DNN Model 2 Fit History using
validation\_data\label{fig:model_11}}
\end{figure}

Using our own validation set also allows us to flexibly measure
performance using tidymodels functions.

The following function keras\_predict() creates prediction results using
the Keras model and preprocessed/baked data from using tidymodels, using
a 50\% probability threshold and works for our binary problem.

\begin{Shaded}
\begin{Highlighting}[]
\NormalTok{keras\_predict }\OtherTok{\textless{}{-}} \ControlFlowTok{function}\NormalTok{(model, baked\_data, response) \{}
\NormalTok{    predictions }\OtherTok{\textless{}{-}} \FunctionTok{predict}\NormalTok{(model, baked\_data)[, }\DecValTok{1}\NormalTok{]}
    \FunctionTok{tibble}\NormalTok{(}\AttributeTok{.pred\_1 =}\NormalTok{ predictions, }\AttributeTok{.pred\_class =} \FunctionTok{if\_else}\NormalTok{(.pred\_1 }\SpecialCharTok{\textless{}} \FloatTok{0.5}\NormalTok{, }\DecValTok{0}\NormalTok{, }\DecValTok{1}\NormalTok{), }\AttributeTok{label =}\NormalTok{ response) }\SpecialCharTok{\%\textgreater{}\%}
        \FunctionTok{mutate}\NormalTok{(}\FunctionTok{across}\NormalTok{(}\FunctionTok{c}\NormalTok{(label, .pred\_class), }\SpecialCharTok{\textasciitilde{}}\FunctionTok{factor}\NormalTok{(.x, }\AttributeTok{levels =} \FunctionTok{c}\NormalTok{(}\DecValTok{1}\NormalTok{, }\DecValTok{0}\NormalTok{))))}
\NormalTok{\}}
\end{Highlighting}
\end{Shaded}

See Table \ref{tbl:val_res} for ``DNN Model 2 Predictions using
validation\_data''

\begin{Shaded}
\begin{Highlighting}[]
\NormalTok{val\_res }\OtherTok{\textless{}{-}} \FunctionTok{keras\_predict}\NormalTok{(dense\_model, amazon\_assess, label\_assess)}
\CommentTok{\# head(val\_res)}
\FunctionTok{kable}\NormalTok{(}\FunctionTok{head}\NormalTok{(val\_res), }\AttributeTok{format =} \StringTok{"simple"}\NormalTok{, }\AttributeTok{caption =} \StringTok{"DNN Model 2 Predictions using validation data}\SpecialCharTok{\textbackslash{}\textbackslash{}}\StringTok{label\{tbl:val\_res\}"}\NormalTok{)}
\end{Highlighting}
\end{Shaded}

\begin{longtable}[]{@{}rll@{}}
\caption{DNN Model 2 Predictions using validation
data\label{tbl:val_res}}\tabularnewline
\toprule
.pred\_1 & .pred\_class & label \\
\midrule
\endfirsthead
\toprule
.pred\_1 & .pred\_class & label \\
\midrule
\endhead
0.0070337 & 0 & 0 \\
0.8914716 & 1 & 1 \\
0.9580745 & 1 & 1 \\
0.3149569 & 0 & 0 \\
0.7441973 & 1 & 1 \\
0.9910595 & 1 & 1 \\
\bottomrule
\end{longtable}

See Table \ref{tbl:val_res_metrics} for ``DNN Model 2 Metrics using
Validation data''

\begin{Shaded}
\begin{Highlighting}[]
\NormalTok{m1 }\OtherTok{\textless{}{-}} \FunctionTok{metrics}\NormalTok{(val\_res, label, .pred\_class)}
\FunctionTok{kable}\NormalTok{(m1, }\AttributeTok{format =} \StringTok{"simple"}\NormalTok{, }\AttributeTok{caption =} \StringTok{"DNN Model 2 Metrics using Validation data}\SpecialCharTok{\textbackslash{}\textbackslash{}}\StringTok{label\{tbl:val\_res\_metrics\}"}\NormalTok{)}
\end{Highlighting}
\end{Shaded}

\begin{longtable}[]{@{}llr@{}}
\caption{DNN Model 2 Metrics using Validation
data\label{tbl:val_res_metrics}}\tabularnewline
\toprule
.metric & .estimator & .estimate \\
\midrule
\endfirsthead
\toprule
.metric & .estimator & .estimate \\
\midrule
\endhead
accuracy & binary & 0.8960983 \\
kap & binary & 0.7870405 \\
\bottomrule
\end{longtable}

``DNN Model 2 Confusion Matrix using Validation data'' Figure
\ref{fig:model_12}

\begin{figure}
\centering
\includegraphics{figures/val_res_conf_mat-1.pdf}
\caption{DNN Model 2 Confusion Matrix using Validation
data\label{fig:model_12}}
\end{figure}

``DNN Model 2 ROC curve using Validation data'' Figure
\ref{fig:model_13}

\begin{figure}
\centering
\includegraphics{figures/val_res_roc-1.pdf}
\caption{DNN Model 2 ROC curve using Validation
data\label{fig:model_13}}
\end{figure}

\newpage

\hypertarget{results-1}{%
\subsection{Results}\label{results-1}}

DNN model results Table \ref{tbl:dnn_results_table}:

\begin{longtable}[]{@{}llrr@{}}
\caption{DNN Model Results\label{tbl:dnn_results_table}}\tabularnewline
\toprule
Index & Method & Accuracy & Loss \\
\midrule
\endfirsthead
\toprule
Index & Method & Accuracy & Loss \\
\midrule
\endhead
1 & BLM & 0.8897064 & NA \\
2 & DNN & 0.8960983 & 0.2790776 \\
\bottomrule
\end{longtable}

\newpage

\hypertarget{model-cnn}{%
\section{Model CNN}\label{model-cnn}}

A CNN is a neural network in which at least one layer is a convolutional
layer. A typical convolutional neural network consists of some
combination of the following layers:

1.Convolutional layers - A layer of a deep neural network in which a
convolutional filter passes along an input matrix.\\
A convolutional operation involves a convolutional filter which is a
matrix having the same rank as the input matrix, but a smaller shape and
a slice of an input matrix. For example, given a 28x28 input matrix, the
filter could be any 2D matrix smaller than 28x28.

For example, in photographic manipulation, all the cells in a
convolutional filter are typically set to a constant pattern of ones and
zeroes. In machine learning, convolutional filters are typically seeded
with random numbers and then the network trains the ideal values.

2.Pooling layers - Reducing a matrix (or matrices) created by an earlier
convolutional layer to a smaller matrix. Pooling usually involves taking
either the maximum or average value across the pooled area. For example,
suppose we have a 3x3 matrix. A pooling operation, just like a
convolutional operation, divides that matrix into slices and then slides
that convolutional operation by strides. For example, suppose the
pooling operation divides the convolutional matrix into 2x2 slices with
a 1x1 stride, then four pooling operations take place. Imagine that each
pooling operation picks the maximum value of the four in that slice.

Pooling helps enforce translational invariance in the input matrix.

Pooling for vision applications is known more formally as spatial
pooling. Time-series applications usually refer to pooling as temporal
pooling. Less formally, pooling is often called subsampling or
downsampling.

3.Dense layers - just a fully connected layer.

Convolutional neural networks have had great success in certain kinds of
problems, especially in image recognition.

\hypertarget{a-first-cnn-model}{%
\subsection{A first CNN model}\label{a-first-cnn-model}}

\begin{Shaded}
\begin{Highlighting}[]
\NormalTok{simple\_cnn\_model }\OtherTok{\textless{}{-}} \FunctionTok{keras\_model\_sequential}\NormalTok{() }\SpecialCharTok{\%\textgreater{}\%}
    \FunctionTok{layer\_embedding}\NormalTok{(}\AttributeTok{input\_dim =}\NormalTok{ max\_words }\SpecialCharTok{+} \DecValTok{1}\NormalTok{, }\AttributeTok{output\_dim =} \DecValTok{16}\NormalTok{, }\AttributeTok{input\_length =}\NormalTok{ max\_length) }\SpecialCharTok{\%\textgreater{}\%}
    \FunctionTok{layer\_batch\_normalization}\NormalTok{() }\SpecialCharTok{\%\textgreater{}\%}
    \FunctionTok{layer\_conv\_1d}\NormalTok{(}\AttributeTok{filter =} \DecValTok{32}\NormalTok{, }\AttributeTok{kernel\_size =} \DecValTok{5}\NormalTok{, }\AttributeTok{activation =} \StringTok{"relu"}\NormalTok{) }\SpecialCharTok{\%\textgreater{}\%}
    \FunctionTok{layer\_max\_pooling\_1d}\NormalTok{(}\AttributeTok{pool\_size =} \DecValTok{2}\NormalTok{) }\SpecialCharTok{\%\textgreater{}\%}
    \FunctionTok{layer\_conv\_1d}\NormalTok{(}\AttributeTok{filter =} \DecValTok{64}\NormalTok{, }\AttributeTok{kernel\_size =} \DecValTok{3}\NormalTok{, }\AttributeTok{activation =} \StringTok{"relu"}\NormalTok{) }\SpecialCharTok{\%\textgreater{}\%}
    \FunctionTok{layer\_global\_max\_pooling\_1d}\NormalTok{() }\SpecialCharTok{\%\textgreater{}\%}
    \FunctionTok{layer\_dense}\NormalTok{(}\AttributeTok{units =} \DecValTok{64}\NormalTok{, }\AttributeTok{activation =} \StringTok{"relu"}\NormalTok{) }\SpecialCharTok{\%\textgreater{}\%}
    \FunctionTok{layer\_dense}\NormalTok{(}\AttributeTok{units =} \DecValTok{1}\NormalTok{, }\AttributeTok{activation =} \StringTok{"sigmoid"}\NormalTok{)}

\NormalTok{simple\_cnn\_model}
\NormalTok{Model}
\NormalTok{Model}\SpecialCharTok{:} \StringTok{"sequential\_2"}
\NormalTok{\_\_\_\_\_\_\_\_\_\_\_\_\_\_\_\_\_\_\_\_\_\_\_\_\_\_\_\_\_\_\_\_\_\_\_\_\_\_\_\_\_\_\_\_\_\_\_\_\_\_\_\_\_\_\_\_\_\_\_\_\_\_\_\_\_\_\_\_\_\_\_\_\_\_\_\_\_\_\_\_}
\FunctionTok{Layer}\NormalTok{ (type)                        Output Shape                    Param }\CommentTok{\#     }
\SpecialCharTok{==}\ErrorTok{==============================================================================}
\FunctionTok{embedding\_2}\NormalTok{ (Embedding)             (None, }\DecValTok{30}\NormalTok{, }\DecValTok{16}\NormalTok{)                  }\DecValTok{320016}      
\NormalTok{\_\_\_\_\_\_\_\_\_\_\_\_\_\_\_\_\_\_\_\_\_\_\_\_\_\_\_\_\_\_\_\_\_\_\_\_\_\_\_\_\_\_\_\_\_\_\_\_\_\_\_\_\_\_\_\_\_\_\_\_\_\_\_\_\_\_\_\_\_\_\_\_\_\_\_\_\_\_\_\_}
\FunctionTok{batch\_normalization}\NormalTok{ (}\FunctionTok{BatchNormaliza}\NormalTok{ (None, }\DecValTok{30}\NormalTok{, }\DecValTok{16}\NormalTok{)                  }\DecValTok{64}          
\NormalTok{\_\_\_\_\_\_\_\_\_\_\_\_\_\_\_\_\_\_\_\_\_\_\_\_\_\_\_\_\_\_\_\_\_\_\_\_\_\_\_\_\_\_\_\_\_\_\_\_\_\_\_\_\_\_\_\_\_\_\_\_\_\_\_\_\_\_\_\_\_\_\_\_\_\_\_\_\_\_\_\_}
\FunctionTok{conv1d\_1}\NormalTok{ (Conv1D)                   (None, }\DecValTok{26}\NormalTok{, }\DecValTok{32}\NormalTok{)                  }\DecValTok{2592}        
\NormalTok{\_\_\_\_\_\_\_\_\_\_\_\_\_\_\_\_\_\_\_\_\_\_\_\_\_\_\_\_\_\_\_\_\_\_\_\_\_\_\_\_\_\_\_\_\_\_\_\_\_\_\_\_\_\_\_\_\_\_\_\_\_\_\_\_\_\_\_\_\_\_\_\_\_\_\_\_\_\_\_\_}
\FunctionTok{max\_pooling1d}\NormalTok{ (MaxPooling1D)        (None, }\DecValTok{13}\NormalTok{, }\DecValTok{32}\NormalTok{)                  }\DecValTok{0}           
\NormalTok{\_\_\_\_\_\_\_\_\_\_\_\_\_\_\_\_\_\_\_\_\_\_\_\_\_\_\_\_\_\_\_\_\_\_\_\_\_\_\_\_\_\_\_\_\_\_\_\_\_\_\_\_\_\_\_\_\_\_\_\_\_\_\_\_\_\_\_\_\_\_\_\_\_\_\_\_\_\_\_\_}
\FunctionTok{conv1d}\NormalTok{ (Conv1D)                     (None, }\DecValTok{11}\NormalTok{, }\DecValTok{64}\NormalTok{)                  }\DecValTok{6208}        
\NormalTok{\_\_\_\_\_\_\_\_\_\_\_\_\_\_\_\_\_\_\_\_\_\_\_\_\_\_\_\_\_\_\_\_\_\_\_\_\_\_\_\_\_\_\_\_\_\_\_\_\_\_\_\_\_\_\_\_\_\_\_\_\_\_\_\_\_\_\_\_\_\_\_\_\_\_\_\_\_\_\_\_}
\FunctionTok{global\_max\_pooling1d}\NormalTok{ (}\FunctionTok{GlobalMaxPool}\NormalTok{ (None, }\DecValTok{64}\NormalTok{)                      }\DecValTok{0}           
\NormalTok{\_\_\_\_\_\_\_\_\_\_\_\_\_\_\_\_\_\_\_\_\_\_\_\_\_\_\_\_\_\_\_\_\_\_\_\_\_\_\_\_\_\_\_\_\_\_\_\_\_\_\_\_\_\_\_\_\_\_\_\_\_\_\_\_\_\_\_\_\_\_\_\_\_\_\_\_\_\_\_\_}
\FunctionTok{dense\_7}\NormalTok{ (Dense)                     (None, }\DecValTok{64}\NormalTok{)                      }\DecValTok{4160}        
\NormalTok{\_\_\_\_\_\_\_\_\_\_\_\_\_\_\_\_\_\_\_\_\_\_\_\_\_\_\_\_\_\_\_\_\_\_\_\_\_\_\_\_\_\_\_\_\_\_\_\_\_\_\_\_\_\_\_\_\_\_\_\_\_\_\_\_\_\_\_\_\_\_\_\_\_\_\_\_\_\_\_\_}
\FunctionTok{dense\_6}\NormalTok{ (Dense)                     (None, }\DecValTok{1}\NormalTok{)                       }\DecValTok{65}          
\SpecialCharTok{==}\ErrorTok{==============================================================================}
\NormalTok{Total params}\SpecialCharTok{:} \DecValTok{333}\NormalTok{,}\DecValTok{105}
\NormalTok{Trainable params}\SpecialCharTok{:} \DecValTok{333}\NormalTok{,}\DecValTok{073}
\NormalTok{Non}\SpecialCharTok{{-}}\NormalTok{trainable params}\SpecialCharTok{:} \DecValTok{32}
\NormalTok{\_\_\_\_\_\_\_\_\_\_\_\_\_\_\_\_\_\_\_\_\_\_\_\_\_\_\_\_\_\_\_\_\_\_\_\_\_\_\_\_\_\_\_\_\_\_\_\_\_\_\_\_\_\_\_\_\_\_\_\_\_\_\_\_\_\_\_\_\_\_\_\_\_\_\_\_\_\_\_\_}
\end{Highlighting}
\end{Shaded}

\begin{Shaded}
\begin{Highlighting}[]
\NormalTok{simple\_cnn\_model }\SpecialCharTok{\%\textgreater{}\%}
    \FunctionTok{compile}\NormalTok{(}\AttributeTok{optimizer =}\NormalTok{ opt, }\AttributeTok{loss =} \StringTok{"binary\_crossentropy"}\NormalTok{, }\AttributeTok{metrics =} \FunctionTok{c}\NormalTok{(}\StringTok{"accuracy"}\NormalTok{))}
\end{Highlighting}
\end{Shaded}

\begin{Shaded}
\begin{Highlighting}[]
\NormalTok{simple\_cnn\_val\_history }\OtherTok{\textless{}{-}}\NormalTok{ simple\_cnn\_model }\SpecialCharTok{\%\textgreater{}\%}
    \FunctionTok{fit}\NormalTok{(}\AttributeTok{x =}\NormalTok{ amazon\_analysis, }\AttributeTok{y =}\NormalTok{ label\_analysis, }\AttributeTok{batch\_size =} \DecValTok{1024}\NormalTok{, }\AttributeTok{epochs =} \DecValTok{7}\NormalTok{, }\AttributeTok{initial\_epoch =} \DecValTok{0}\NormalTok{,}
        \AttributeTok{validation\_data =} \FunctionTok{list}\NormalTok{(amazon\_assess, label\_assess), }\AttributeTok{verbose =} \DecValTok{2}\NormalTok{)}

\NormalTok{simple\_cnn\_val\_history}

\NormalTok{Final }\FunctionTok{epoch}\NormalTok{ (plot to see history)}\SpecialCharTok{:}
\NormalTok{        loss}\SpecialCharTok{:} \FloatTok{0.2097}
\NormalTok{    accuracy}\SpecialCharTok{:} \FloatTok{0.9192}
\NormalTok{    val\_loss}\SpecialCharTok{:} \FloatTok{0.2526}
\NormalTok{val\_accuracy}\SpecialCharTok{:} \FloatTok{0.8983} 
\end{Highlighting}
\end{Shaded}

``CNN Model Fit History using validation\_data'' Figure
\ref{fig:model_14}

\begin{figure}
\centering
\includegraphics{figures/simple_cnn_model_fit_val_d_hist-1.pdf}
\caption{CNN Model Fit History using
validation\_data\label{fig:model_14}}
\end{figure}

See Table \ref{tbl:simple_cnn_val_res} for ``CNN Model Predictions using
validation data''

\begin{Shaded}
\begin{Highlighting}[]
\NormalTok{simple\_cnn\_val\_res }\OtherTok{\textless{}{-}} \FunctionTok{keras\_predict}\NormalTok{(simple\_cnn\_model, amazon\_assess, label\_assess)}
\CommentTok{\# head(simple\_cnn\_val\_res)}
\FunctionTok{kable}\NormalTok{(}\FunctionTok{head}\NormalTok{(simple\_cnn\_val\_res), }\AttributeTok{format =} \StringTok{"simple"}\NormalTok{, }\AttributeTok{caption =} \StringTok{"CNN Model Predictions using validation data}\SpecialCharTok{\textbackslash{}\textbackslash{}}\StringTok{label\{tbl:simple\_cnn\_val\_res\}"}\NormalTok{)}
\end{Highlighting}
\end{Shaded}

\begin{longtable}[]{@{}rll@{}}
\caption{CNN Model Predictions using validation
data\label{tbl:simple_cnn_val_res}}\tabularnewline
\toprule
.pred\_1 & .pred\_class & label \\
\midrule
\endfirsthead
\toprule
.pred\_1 & .pred\_class & label \\
\midrule
\endhead
0.0036336 & 0 & 0 \\
0.9667417 & 1 & 1 \\
0.9940560 & 1 & 1 \\
0.1679935 & 0 & 0 \\
0.8921552 & 1 & 1 \\
0.9995990 & 1 & 1 \\
\bottomrule
\end{longtable}

See Table \ref{tbl:simple_cnn_val_res_metrics} for ``CNN Model Metrics
using validation data''

\begin{Shaded}
\begin{Highlighting}[]
\NormalTok{m2 }\OtherTok{\textless{}{-}} \FunctionTok{metrics}\NormalTok{(simple\_cnn\_val\_res, label, .pred\_class)}
\FunctionTok{kable}\NormalTok{(m2, }\AttributeTok{format =} \StringTok{"simple"}\NormalTok{, }\AttributeTok{caption =} \StringTok{"CNN Model Metrics using validation data}\SpecialCharTok{\textbackslash{}\textbackslash{}}\StringTok{label\{tbl:simple\_cnn\_val\_res\_metrics\}"}\NormalTok{)}
\end{Highlighting}
\end{Shaded}

\begin{longtable}[]{@{}llr@{}}
\caption{CNN Model Metrics using validation
data\label{tbl:simple_cnn_val_res_metrics}}\tabularnewline
\toprule
.metric & .estimator & .estimate \\
\midrule
\endfirsthead
\toprule
.metric & .estimator & .estimate \\
\midrule
\endhead
accuracy & binary & 0.8982586 \\
kap & binary & 0.7911622 \\
\bottomrule
\end{longtable}

``CNN Model Confusion Matrix using validation\_data'' Figure
\ref{fig:model_15}

\begin{figure}
\centering
\includegraphics{figures/simple_cnn_val_res_conf_mat-1.pdf}
\caption{CNN Model Confusion Matrix using
validation\_data\label{fig:model_15}}
\end{figure}

``CNN Model ROC curve using validation\_data'' Figure \ref{fig:model_16}

\begin{figure}
\centering
\includegraphics{figures/simple_cnn_val_res_roc-1.pdf}
\caption{CNN Model ROC curve using validation\_data\label{fig:model_16}}
\end{figure}

\newpage

\hypertarget{results-2}{%
\subsection{Results}\label{results-2}}

CNN model results Table \ref{tbl:cnn_results_table}:

\begin{longtable}[]{@{}llrr@{}}
\caption{CNN Model Results\label{tbl:cnn_results_table}}\tabularnewline
\toprule
Index & Method & Accuracy & Loss \\
\midrule
\endfirsthead
\toprule
Index & Method & Accuracy & Loss \\
\midrule
\endhead
1 & BLM & 0.8897064 & NA \\
2 & DNN & 0.8960983 & 0.2790776 \\
3 & CNN & 0.8982586 & 0.2526298 \\
\bottomrule
\end{longtable}

\newpage

\hypertarget{model-sepcnn}{%
\section{Model sepCNN}\label{model-sepcnn}}

A convolutional neural network architecture based on
\href{https://github.com/tensorflow/tpu/tree/master/models/experimental/inception}{Inception},
but where Inception modules are replaced with depthwise separable
convolutions. Also known as Xception.

A depthwise separable convolution (also abbreviated as separable
convolution) factors a standard 3-D convolution into two separate
convolution operations that are more computationally efficient: first, a
depthwise convolution, with a depth of 1 (n x n x 1), and then second, a
pointwise convolution, with length and width of 1 (1 x 1 x n).

To learn more, see \href{https://arxiv.org/pdf/1610.02357.pdf}{Xception:
Deep Learning with Depthwise Separable Convolutions}.

\hypertarget{a-first-sepcnn-model}{%
\subsection{A first sepCNN model}\label{a-first-sepcnn-model}}

\begin{Shaded}
\begin{Highlighting}[]
\NormalTok{sep\_cnn\_model }\OtherTok{\textless{}{-}} \FunctionTok{keras\_model\_sequential}\NormalTok{() }\SpecialCharTok{\%\textgreater{}\%}
    \FunctionTok{layer\_embedding}\NormalTok{(}\AttributeTok{input\_dim =}\NormalTok{ max\_words }\SpecialCharTok{+} \DecValTok{1}\NormalTok{, }\AttributeTok{output\_dim =} \DecValTok{16}\NormalTok{, }\AttributeTok{input\_length =}\NormalTok{ max\_length) }\SpecialCharTok{\%\textgreater{}\%}
    \CommentTok{\# layer\_batch\_normalization() \%\textgreater{}\%}
\FunctionTok{layer\_dropout}\NormalTok{(}\FloatTok{0.2}\NormalTok{) }\SpecialCharTok{\%\textgreater{}\%}
    \FunctionTok{layer\_separable\_conv\_1d}\NormalTok{(}\AttributeTok{filter =} \DecValTok{32}\NormalTok{, }\AttributeTok{kernel\_size =} \DecValTok{5}\NormalTok{, }\AttributeTok{activation =} \StringTok{"relu"}\NormalTok{) }\SpecialCharTok{\%\textgreater{}\%}
    \FunctionTok{layer\_separable\_conv\_1d}\NormalTok{(}\AttributeTok{filter =} \DecValTok{32}\NormalTok{, }\AttributeTok{kernel\_size =} \DecValTok{5}\NormalTok{, }\AttributeTok{activation =} \StringTok{"relu"}\NormalTok{) }\SpecialCharTok{\%\textgreater{}\%}
    \FunctionTok{layer\_max\_pooling\_1d}\NormalTok{(}\AttributeTok{pool\_size =} \DecValTok{2}\NormalTok{) }\SpecialCharTok{\%\textgreater{}\%}
    \FunctionTok{layer\_separable\_conv\_1d}\NormalTok{(}\AttributeTok{filter =} \DecValTok{64}\NormalTok{, }\AttributeTok{kernel\_size =} \DecValTok{5}\NormalTok{, }\AttributeTok{activation =} \StringTok{"relu"}\NormalTok{) }\SpecialCharTok{\%\textgreater{}\%}
    \FunctionTok{layer\_separable\_conv\_1d}\NormalTok{(}\AttributeTok{filter =} \DecValTok{64}\NormalTok{, }\AttributeTok{kernel\_size =} \DecValTok{5}\NormalTok{, }\AttributeTok{activation =} \StringTok{"relu"}\NormalTok{) }\SpecialCharTok{\%\textgreater{}\%}
    \FunctionTok{layer\_global\_average\_pooling\_1d}\NormalTok{() }\SpecialCharTok{\%\textgreater{}\%}
    \FunctionTok{layer\_dropout}\NormalTok{(}\FloatTok{0.2}\NormalTok{) }\SpecialCharTok{\%\textgreater{}\%}
    \CommentTok{\# layer\_dense(units = 64, activation = \textquotesingle{}relu\textquotesingle{}) \%\textgreater{}\%}
\FunctionTok{layer\_dense}\NormalTok{(}\AttributeTok{units =} \DecValTok{1}\NormalTok{, }\AttributeTok{activation =} \StringTok{"sigmoid"}\NormalTok{)}

\NormalTok{sep\_cnn\_model}
\NormalTok{Model}
\NormalTok{Model}\SpecialCharTok{:} \StringTok{"sequential\_3"}
\NormalTok{\_\_\_\_\_\_\_\_\_\_\_\_\_\_\_\_\_\_\_\_\_\_\_\_\_\_\_\_\_\_\_\_\_\_\_\_\_\_\_\_\_\_\_\_\_\_\_\_\_\_\_\_\_\_\_\_\_\_\_\_\_\_\_\_\_\_\_\_\_\_\_\_\_\_\_\_\_\_\_\_}
\FunctionTok{Layer}\NormalTok{ (type)                        Output Shape                    Param }\CommentTok{\#     }
\SpecialCharTok{==}\ErrorTok{==============================================================================}
\FunctionTok{embedding\_3}\NormalTok{ (Embedding)             (None, }\DecValTok{30}\NormalTok{, }\DecValTok{16}\NormalTok{)                  }\DecValTok{320016}      
\NormalTok{\_\_\_\_\_\_\_\_\_\_\_\_\_\_\_\_\_\_\_\_\_\_\_\_\_\_\_\_\_\_\_\_\_\_\_\_\_\_\_\_\_\_\_\_\_\_\_\_\_\_\_\_\_\_\_\_\_\_\_\_\_\_\_\_\_\_\_\_\_\_\_\_\_\_\_\_\_\_\_\_}
\FunctionTok{dropout\_3}\NormalTok{ (Dropout)                 (None, }\DecValTok{30}\NormalTok{, }\DecValTok{16}\NormalTok{)                  }\DecValTok{0}           
\NormalTok{\_\_\_\_\_\_\_\_\_\_\_\_\_\_\_\_\_\_\_\_\_\_\_\_\_\_\_\_\_\_\_\_\_\_\_\_\_\_\_\_\_\_\_\_\_\_\_\_\_\_\_\_\_\_\_\_\_\_\_\_\_\_\_\_\_\_\_\_\_\_\_\_\_\_\_\_\_\_\_\_}
\FunctionTok{separable\_conv1d\_3}\NormalTok{ (}\FunctionTok{SeparableConv1D}\NormalTok{ (None, }\DecValTok{26}\NormalTok{, }\DecValTok{32}\NormalTok{)                  }\DecValTok{624}         
\NormalTok{\_\_\_\_\_\_\_\_\_\_\_\_\_\_\_\_\_\_\_\_\_\_\_\_\_\_\_\_\_\_\_\_\_\_\_\_\_\_\_\_\_\_\_\_\_\_\_\_\_\_\_\_\_\_\_\_\_\_\_\_\_\_\_\_\_\_\_\_\_\_\_\_\_\_\_\_\_\_\_\_}
\FunctionTok{separable\_conv1d\_2}\NormalTok{ (}\FunctionTok{SeparableConv1D}\NormalTok{ (None, }\DecValTok{22}\NormalTok{, }\DecValTok{32}\NormalTok{)                  }\DecValTok{1216}        
\NormalTok{\_\_\_\_\_\_\_\_\_\_\_\_\_\_\_\_\_\_\_\_\_\_\_\_\_\_\_\_\_\_\_\_\_\_\_\_\_\_\_\_\_\_\_\_\_\_\_\_\_\_\_\_\_\_\_\_\_\_\_\_\_\_\_\_\_\_\_\_\_\_\_\_\_\_\_\_\_\_\_\_}
\FunctionTok{max\_pooling1d\_1}\NormalTok{ (MaxPooling1D)      (None, }\DecValTok{11}\NormalTok{, }\DecValTok{32}\NormalTok{)                  }\DecValTok{0}           
\NormalTok{\_\_\_\_\_\_\_\_\_\_\_\_\_\_\_\_\_\_\_\_\_\_\_\_\_\_\_\_\_\_\_\_\_\_\_\_\_\_\_\_\_\_\_\_\_\_\_\_\_\_\_\_\_\_\_\_\_\_\_\_\_\_\_\_\_\_\_\_\_\_\_\_\_\_\_\_\_\_\_\_}
\FunctionTok{separable\_conv1d\_1}\NormalTok{ (}\FunctionTok{SeparableConv1D}\NormalTok{ (None, }\DecValTok{7}\NormalTok{, }\DecValTok{64}\NormalTok{)                   }\DecValTok{2272}        
\NormalTok{\_\_\_\_\_\_\_\_\_\_\_\_\_\_\_\_\_\_\_\_\_\_\_\_\_\_\_\_\_\_\_\_\_\_\_\_\_\_\_\_\_\_\_\_\_\_\_\_\_\_\_\_\_\_\_\_\_\_\_\_\_\_\_\_\_\_\_\_\_\_\_\_\_\_\_\_\_\_\_\_}
\FunctionTok{separable\_conv1d}\NormalTok{ (SeparableConv1D)  (None, }\DecValTok{3}\NormalTok{, }\DecValTok{64}\NormalTok{)                   }\DecValTok{4480}        
\NormalTok{\_\_\_\_\_\_\_\_\_\_\_\_\_\_\_\_\_\_\_\_\_\_\_\_\_\_\_\_\_\_\_\_\_\_\_\_\_\_\_\_\_\_\_\_\_\_\_\_\_\_\_\_\_\_\_\_\_\_\_\_\_\_\_\_\_\_\_\_\_\_\_\_\_\_\_\_\_\_\_\_}
\FunctionTok{global\_average\_pooling1d}\NormalTok{ (}\FunctionTok{GlobalAve}\NormalTok{ (None, }\DecValTok{64}\NormalTok{)                      }\DecValTok{0}           
\NormalTok{\_\_\_\_\_\_\_\_\_\_\_\_\_\_\_\_\_\_\_\_\_\_\_\_\_\_\_\_\_\_\_\_\_\_\_\_\_\_\_\_\_\_\_\_\_\_\_\_\_\_\_\_\_\_\_\_\_\_\_\_\_\_\_\_\_\_\_\_\_\_\_\_\_\_\_\_\_\_\_\_}
\FunctionTok{dropout\_2}\NormalTok{ (Dropout)                 (None, }\DecValTok{64}\NormalTok{)                      }\DecValTok{0}           
\NormalTok{\_\_\_\_\_\_\_\_\_\_\_\_\_\_\_\_\_\_\_\_\_\_\_\_\_\_\_\_\_\_\_\_\_\_\_\_\_\_\_\_\_\_\_\_\_\_\_\_\_\_\_\_\_\_\_\_\_\_\_\_\_\_\_\_\_\_\_\_\_\_\_\_\_\_\_\_\_\_\_\_}
\FunctionTok{dense\_8}\NormalTok{ (Dense)                     (None, }\DecValTok{1}\NormalTok{)                       }\DecValTok{65}          
\SpecialCharTok{==}\ErrorTok{==============================================================================}
\NormalTok{Total params}\SpecialCharTok{:} \DecValTok{328}\NormalTok{,}\DecValTok{673}
\NormalTok{Trainable params}\SpecialCharTok{:} \DecValTok{328}\NormalTok{,}\DecValTok{673}
\NormalTok{Non}\SpecialCharTok{{-}}\NormalTok{trainable params}\SpecialCharTok{:} \DecValTok{0}
\NormalTok{\_\_\_\_\_\_\_\_\_\_\_\_\_\_\_\_\_\_\_\_\_\_\_\_\_\_\_\_\_\_\_\_\_\_\_\_\_\_\_\_\_\_\_\_\_\_\_\_\_\_\_\_\_\_\_\_\_\_\_\_\_\_\_\_\_\_\_\_\_\_\_\_\_\_\_\_\_\_\_\_}
\end{Highlighting}
\end{Shaded}

\begin{Shaded}
\begin{Highlighting}[]
\CommentTok{\# opt \textless{}{-} optimizer\_sgd(lr = 0.001, decay = 1e{-}6) opt \textless{}{-} optimizer\_adam() opt \textless{}{-}}
\CommentTok{\# optimizer\_sgd()}
\NormalTok{opt }\OtherTok{\textless{}{-}} \FunctionTok{optimizer\_adam}\NormalTok{(}\AttributeTok{lr =} \FloatTok{1e{-}04}\NormalTok{, }\AttributeTok{decay =} \FloatTok{1e{-}06}\NormalTok{)}
\NormalTok{sep\_cnn\_model }\SpecialCharTok{\%\textgreater{}\%}
    \FunctionTok{compile}\NormalTok{(}\AttributeTok{optimizer =}\NormalTok{ opt, }\AttributeTok{loss =} \StringTok{"binary\_crossentropy"}\NormalTok{, }\AttributeTok{metrics =} \FunctionTok{c}\NormalTok{(}\StringTok{"accuracy"}\NormalTok{))}
\end{Highlighting}
\end{Shaded}

Here we use a
\href{https://tensorflow.rstudio.com/guide/keras/guide_keras/\#sts=Callbacks}{callback}.

keras::callback\_early\_stopping: Interrupt training when validation
performance has stopped improving.\\
patience: number of epochs with no improvement after which training will
be stopped.

Try ??keras::callback\_early\_stopping

\begin{Shaded}
\begin{Highlighting}[]
\NormalTok{sep\_cnn\_val\_history }\OtherTok{\textless{}{-}}\NormalTok{ sep\_cnn\_model }\SpecialCharTok{\%\textgreater{}\%}
  \FunctionTok{fit}\NormalTok{(}
    \AttributeTok{x =}\NormalTok{ amazon\_analysis,}
    \AttributeTok{y =}\NormalTok{ label\_analysis,}
    \AttributeTok{batch\_size =} \DecValTok{128}\NormalTok{,}
    \AttributeTok{epochs =} \DecValTok{20}\NormalTok{,}
    \AttributeTok{initial\_epoch =} \DecValTok{0}\NormalTok{,}
    \AttributeTok{validation\_data =} \FunctionTok{list}\NormalTok{(amazon\_assess, label\_assess),}
    \AttributeTok{callbacks =} \FunctionTok{list}\NormalTok{(}\FunctionTok{callback\_early\_stopping}\NormalTok{(}
        \AttributeTok{monitor=}\StringTok{\textquotesingle{}val\_loss\textquotesingle{}}\NormalTok{, }\AttributeTok{patience=}\DecValTok{2}\NormalTok{)),}
    \AttributeTok{verbose =} \DecValTok{2}
\NormalTok{  )}

\NormalTok{sep\_cnn\_val\_history}
\end{Highlighting}
\end{Shaded}

\begin{verbatim}
## 
## Final epoch (plot to see history):
##         loss: 0.1894
##     accuracy: 0.9279
##     val_loss: 0.2417
## val_accuracy: 0.9052
\end{verbatim}

``sepCNN Model Fit History using validation\_data'' Figure
\ref{fig:model_17}

\begin{figure}
\centering
\includegraphics{figures/sep_cnn_model_fit_val_d_hist-1.pdf}
\caption{sepCNN Model Fit History using
validation\_data\label{fig:model_17}}
\end{figure}

See Table \ref{tbl:sep_cnn_val_res} for ``sepCNN Model Predictions using
validation data''

\begin{Shaded}
\begin{Highlighting}[]
\NormalTok{sep\_cnn\_val\_res }\OtherTok{\textless{}{-}} \FunctionTok{keras\_predict}\NormalTok{(sep\_cnn\_model, amazon\_assess, label\_assess)}
\CommentTok{\# head(sep\_cnn\_val\_res)}
\FunctionTok{kable}\NormalTok{(}\FunctionTok{head}\NormalTok{(sep\_cnn\_val\_res), }\AttributeTok{format=}\StringTok{"simple"}\NormalTok{, }\AttributeTok{caption=}\StringTok{"sepCNN Model Predictions using validation data}\SpecialCharTok{\textbackslash{}\textbackslash{}}\StringTok{label\{tbl:sep\_cnn\_val\_res\}"}\NormalTok{)}
\end{Highlighting}
\end{Shaded}

\begin{longtable}[]{@{}rll@{}}
\caption{sepCNN Model Predictions using validation
data\label{tbl:sep_cnn_val_res}}\tabularnewline
\toprule
.pred\_1 & .pred\_class & label \\
\midrule
\endfirsthead
\toprule
.pred\_1 & .pred\_class & label \\
\midrule
\endhead
0.0000648 & 0 & 0 \\
0.9813509 & 1 & 1 \\
0.9909515 & 1 & 1 \\
0.2639050 & 0 & 0 \\
0.8420113 & 1 & 1 \\
0.9995446 & 1 & 1 \\
\bottomrule
\end{longtable}

See Table \ref{tbl:sep_cnn_val_res_metrics} for ``sepCNN Model Metrics
using validation data''

\begin{Shaded}
\begin{Highlighting}[]
\NormalTok{m3 }\OtherTok{\textless{}{-}} \FunctionTok{metrics}\NormalTok{(sep\_cnn\_val\_res, label, .pred\_class)}
\FunctionTok{kable}\NormalTok{(m3, }\AttributeTok{format=}\StringTok{"simple"}\NormalTok{, }\AttributeTok{caption=}\StringTok{"sepCNN Model Metrics using validation data}\SpecialCharTok{\textbackslash{}\textbackslash{}}\StringTok{label\{tbl:sep\_cnn\_val\_res\_metrics\}"}\NormalTok{)}
\end{Highlighting}
\end{Shaded}

\begin{longtable}[]{@{}llr@{}}
\caption{sepCNN Model Metrics using validation
data\label{tbl:sep_cnn_val_res_metrics}}\tabularnewline
\toprule
.metric & .estimator & .estimate \\
\midrule
\endfirsthead
\toprule
.metric & .estimator & .estimate \\
\midrule
\endhead
accuracy & binary & 0.9051675 \\
kap & binary & 0.8045877 \\
\bottomrule
\end{longtable}

``sepCNN Model Confusion Matrix using validation\_data'' Figure
\ref{fig:model_18}

\begin{figure}
\centering
\includegraphics{figures/sep_cnn_val_res_conf_mat-1.pdf}
\caption{sepCNN Model Confusion Matrix using
validation\_data\label{fig:model_18}}
\end{figure}

``sepCNN Model ROC curve using validation\_data'' Figure
\ref{fig:model_19}

\begin{figure}
\centering
\includegraphics{figures/sep_cnn_val_res_roc-1.pdf}
\caption{sepCNN Model ROC curve using
validation\_data\label{fig:model_19}}
\end{figure}

\newpage

\hypertarget{results-3}{%
\subsection{Results}\label{results-3}}

sepCNN model results Table \ref{tbl:sep_cnn_results_table}:

\begin{longtable}[]{@{}llrr@{}}
\caption{sepCNN Model
Results\label{tbl:sep_cnn_results_table}}\tabularnewline
\toprule
Index & Method & Accuracy & Loss \\
\midrule
\endfirsthead
\toprule
Index & Method & Accuracy & Loss \\
\midrule
\endhead
1 & BLM & 0.8897064 & NA \\
2 & DNN & 0.8960983 & 0.2790776 \\
3 & CNN & 0.8982586 & 0.2526298 \\
4 & sepCNN & 0.9051675 & 0.2408866 \\
\bottomrule
\end{longtable}

\newpage

\hypertarget{model-bert}{%
\section{Model BERT}\label{model-bert}}

\hypertarget{about-bert}{%
\subsection{About BERT}\label{about-bert}}

In this model we will fine-tune BERT to perform sentiment analysis on
our dataset.

\href{https://arxiv.org/abs/1810.04805}{BERT} and other Transformer
encoder architectures have been wildly successful on a variety of tasks
in NLP (natural language processing). They compute vector-space
representations of natural language that are suitable for use in deep
learning models. The BERT family of models uses the Transformer encoder
architecture to process each token of input text in the full context of
all tokens before and after, hence the name: Bidirectional Encoder
Representations from Transformers.

BERT models are usually pre-trained on a large corpus of text, then
fine-tuned for specific tasks.

\hypertarget{references-1}{%
\subsection{References}\label{references-1}}

\href{https://www.tensorflow.org/}{Tensorflow} is an end-to-end open
source platform for machine learning. It has a comprehensive, flexible
ecosystem of tools, libraries and community resources that lets
researchers push the state-of-the-art in ML and developers easily build
and deploy ML powered applications.

The \href{https://tfhub.dev/}{TensorFlow Hub} lets you search and
discover hundreds of trained, ready-to-deploy machine learning models in
one place.

\href{https://tensorflow.rstudio.com/}{Tensorflow for R} provides an R
interface for Tensorflow.

\hypertarget{loading-csv-data}{%
\subsection{Loading CSV data}\label{loading-csv-data}}

Load CSV data from a file into a TensorFlow Dataset using tfdatasets.

\hypertarget{setup}{%
\subsection{Setup}\label{setup}}

\begin{Shaded}
\begin{Highlighting}[]
\FunctionTok{library}\NormalTok{(keras)}
\FunctionTok{library}\NormalTok{(tfdatasets)}
\FunctionTok{library}\NormalTok{(reticulate)}
\FunctionTok{library}\NormalTok{(tidyverse)}
\FunctionTok{library}\NormalTok{(lubridate)}
\FunctionTok{library}\NormalTok{(tfhub)}

\CommentTok{\# A dependency of the preprocessing for BERT inputs pip install {-}q {-}U}
\CommentTok{\# tensorflow{-}text}
\FunctionTok{import}\NormalTok{(}\StringTok{"tensorflow\_text"}\NormalTok{)}
\FunctionTok{Module}\NormalTok{(tensorflow\_text)}

\CommentTok{\# You will use the AdamW optimizer from tensorflow/models.  pip install {-}q}
\CommentTok{\# tf{-}models{-}official to create AdamW optimizer}
\NormalTok{o\_nlp }\OtherTok{\textless{}{-}} \FunctionTok{import}\NormalTok{(}\StringTok{"official.nlp"}\NormalTok{)}

\FunctionTok{Sys.setenv}\NormalTok{(}\AttributeTok{TFHUB\_CACHE\_DIR =} \StringTok{"C:/Users/bijoor/.cache/tfhub\_modules"}\NormalTok{)}
\FunctionTok{Sys.getenv}\NormalTok{(}\StringTok{"TFHUB\_CACHE\_DIR"}\NormalTok{)}
\NormalTok{[}\DecValTok{1}\NormalTok{] }\StringTok{"C:/Users/bijoor/.cache/tfhub\_modules"}
\end{Highlighting}
\end{Shaded}

You could load this using read.csv, and pass the arrays to TensorFlow.
If you need to scale up to a large set of files, or need a loader that
integrates with TensorFlow and tfdatasets then use the
make\_csv\_dataset function:

Now read the CSV data from the file and create a dataset.

\hypertarget{make-datasets}{%
\subsection{Make datasets}\label{make-datasets}}

\begin{Shaded}
\begin{Highlighting}[]
\NormalTok{train\_file\_path }\OtherTok{\textless{}{-}} \FunctionTok{file.path}\NormalTok{(}\StringTok{"amazon\_review\_polarity\_csv/amazon\_train.csv"}\NormalTok{)}

\NormalTok{batch\_size }\OtherTok{\textless{}{-}} \DecValTok{32}

\NormalTok{train\_dataset }\OtherTok{\textless{}{-}} \FunctionTok{make\_csv\_dataset}\NormalTok{(train\_file\_path, }\AttributeTok{field\_delim =} \StringTok{","}\NormalTok{, }\AttributeTok{batch\_size =}\NormalTok{ batch\_size,}
    \AttributeTok{column\_names =} \FunctionTok{list}\NormalTok{(}\StringTok{"label"}\NormalTok{, }\StringTok{"title"}\NormalTok{, }\StringTok{"text"}\NormalTok{), }\AttributeTok{label\_name =} \StringTok{"label"}\NormalTok{, }\AttributeTok{select\_columns =} \FunctionTok{list}\NormalTok{(}\StringTok{"label"}\NormalTok{,}
        \StringTok{"text"}\NormalTok{), }\AttributeTok{num\_epochs =} \DecValTok{1}\NormalTok{)}


\NormalTok{train\_dataset }\SpecialCharTok{\%\textgreater{}\%}
\NormalTok{    reticulate}\SpecialCharTok{::}\FunctionTok{as\_iterator}\NormalTok{() }\SpecialCharTok{\%\textgreater{}\%}
\NormalTok{    reticulate}\SpecialCharTok{::}\FunctionTok{iter\_next}\NormalTok{()  }\CommentTok{\#\%\textgreater{}\% }
\NormalTok{[[}\DecValTok{1}\NormalTok{]]}
\FunctionTok{OrderedDict}\NormalTok{([(}\StringTok{\textquotesingle{}text\textquotesingle{}}\NormalTok{, }\SpecialCharTok{\textless{}}\NormalTok{tf.Tensor}\SpecialCharTok{:} \AttributeTok{shape=}\NormalTok{(}\DecValTok{32}\NormalTok{,), }\AttributeTok{dtype=}\NormalTok{string, }\AttributeTok{numpy=}
\FunctionTok{array}\NormalTok{([b}\StringTok{\textquotesingle{}I am years old and have read many books like this one before I found that it said little while being very erudite If I was younger I would probably be a bit wowed and perhaps confused by it Maybe if I was in my s I would have found it a better read At my level of development I learned little I didn t already know and found little content\textquotesingle{}}\NormalTok{,}
\NormalTok{       b}\StringTok{\textquotesingle{}I would not recommend this flashlight to anyone who actually wants to be able to see in the dark It is really dim and you can hardly see an inch in front of you with this flashlight My recommendation don t waste your money on this\textquotesingle{}}\NormalTok{,}
\NormalTok{       b}\StringTok{\textquotesingle{}Has tons of game storage but I bought a starter pack initially which hardly held more than the game I wish this was just a bit bigger If you have a travel case on the game it s then hard to fit all the adapters and accessories in this case too\textquotesingle{}}\NormalTok{,}
\NormalTok{       b}\StringTok{\textquotesingle{}An abortion Read other star reviews for more depth but please please do not read anything translated by Andrew Hurley It s sad that people don t realize there are other options out there probably because this complete fictions pop up first on Amazon Start with Labyrinths Do not read anything translated by Andrew Hurley read Di Giovanni Yeats\textquotesingle{}}\NormalTok{,}
\NormalTok{       b}\StringTok{\textquotesingle{}I have loved Wrinkle in Time since I first read it in grade school It is a wonderful book that never gets old I thought that the audio version would be perfect for my iPod I have to say while I love the author s books her reading this story is horrible As the another reviewer said she has a lisp or speech impediment or something It is almost impossible to finish this audio version Her reading is also very slow in spots especially when she is reading the parts of Mrs Which I would love to hear this read by a professional reader or an actor Perhaps now that she has passed away someone will revisit making the audio version of Wrinkle in Time\textquotesingle{}}\NormalTok{,}
\NormalTok{       b}\StringTok{\textquotesingle{}I hate sounding like a commercial but this product is amazing It melted the mastic into a pool of dirty water that I could clean up with a sponge No scraping at all It not only cleaned up the mastic but it also cleaned up the mess I made attempting to use other mastic removing techniques D And best of all NO FUMES My cat and I were comfortably in the apartment through the whole process\textquotesingle{}}\NormalTok{,}
\NormalTok{       b}\StringTok{\textquotesingle{}This book is unusual for a Holocaust book because none of the protagonists are wholly good or entirely bad The villain is capable of love and the female lead character is unable to love Good men are damaged goods and even the innocent next generation is disabled It definitely is worth reading\textquotesingle{}}\NormalTok{,}
\NormalTok{       b}\StringTok{\textquotesingle{}This product is strange First off every time i start a game I have to unplug the card to get a reaction from the actual playstation controller I dunno if this is the products fault or the console itself but it s rather irritating And second I noticed that some of my games don t even save That one I m pretty sure I can blame the card for Personally I wouldn t buy this card again\textquotesingle{}}\NormalTok{,}
\NormalTok{       b}\StringTok{\textquotesingle{}I used this figure for parts for a custom Darth Sion figure I made It actually makes a great Darth Sion if you repaint it I recommend using the Anakin burn damage head\textquotesingle{}}\NormalTok{,}
\NormalTok{       b}\StringTok{\textquotesingle{}I have many problems with this book First it is not a reprint of a old book but a photocopy of the book It is hard to read the text The original book had some color photos but in this photocopy those photos are missing Imagine in a book about matching color and grain patterns with no photos\textquotesingle{}}\NormalTok{,}
\NormalTok{       b}\StringTok{\textquotesingle{}Had used it for about months and within the last months I was thinking to throw it to the cemend floor from the roof of my house after every time I used it And today I did It was so hard to load the paper correctly lot of wired parts were blocking the paper I needed to spend around min to feedone piece of paper to the tray till this piece of paper looked like a piece of toilet paper after I tried all my effort to finish the fax the poeple on the other end might receive couple pieces of blank paper I d rather to throw this junk to a garbage can than donate it to anyone who really needs a fax machine because a garbage can is where this machine really belongs to trust me\textquotesingle{}}\NormalTok{,}
\NormalTok{       b}\StringTok{\textquotesingle{}The newest remake of this movie doesn t do justice to the story This is an all time great and my year old loves this movie just as much as his dad He had a ton of questions and we spoke about the movie for weeks after we saw it We continue to watch it together again and again\textquotesingle{}}\NormalTok{,}
\NormalTok{       b}\StringTok{\textquotesingle{}I have a nine foot tree The bottom three pieces needed to go in their own bag and the top two fit in one For the price I suggest buying an extra bag so you don t have to stuff the tree in\textquotesingle{}}\NormalTok{,}
\NormalTok{       b}\StringTok{\textquotesingle{}I have recently purchased this wine making book It contains many receipies for many diferent types and styles of wines I have found it to be very helpful in my efforts to making good sound wines\textquotesingle{}}\NormalTok{,}
\NormalTok{       b}\StringTok{\textquotesingle{}I think this is a very comprehensive report on reflux problems I have a sonwith Barrett s Esophagus and he seems to ignore the importance of diet He eatsfast food often and does not pretend to be on a diet I bought this as a gift for him and think it will help him immensely It goes all the way explaining the different drugs and their benefits The menusare a big help for those who want to prepare meals at home It is well worth the price in my opinion\textquotesingle{}}\NormalTok{,}
\NormalTok{       b}\StringTok{\textquotesingle{}If I could rate this game lower than star I would You need an NVIDIA Video card to play this game My month old PC doesn t support Pixel Shader so this game won t run on it My yr old son purchased the game with his own money I hope Lego and Eidos realize they have just lost a young customer (and his parents)Be very careful to check your video card before purchasing this game\textquotesingle{}}\NormalTok{,}
\NormalTok{       b}\StringTok{\textquotesingle{}I really wanted to love these treats They re healthy and smell wonderful but my dogs will NOT touch them I ve tried everything and neither one a lab mix and a wheaton Havenese mix will not touch them Not sure why because they LOVE the duck wrapped sweat potato treats I ve gotten them beef sticks to chew on and they re just ok with those maybe it s my dogs\textquotesingle{}}\NormalTok{,}
\NormalTok{       b}\StringTok{\textquotesingle{}Having read most of her books Agatha Christie still outwits me Very seldom one can guess the final outcome of her books and this is one of the very typical specimens Actually all her books offer surprises and this particular one offers nothing less I would say it s another masterpiece of hers though the revelation of the real murderer actually saddens my heart (Yes very much )I agree with one of the reviews that say the best books of Agatha Christie are And Then There Were None and The Murder On The Orient Express (Try The Secret Adversary also) But the reviewer made a mistake by saying The Murder of Roger Ackroyd is the first book of Poirot actually it s not that should be The Mystery Affair at Styles\textquotesingle{}}\NormalTok{,}
\NormalTok{       b}\StringTok{\textquotesingle{}My lb jack russel destroyed two of these in hours of purchase don t waste ur money regular tennis balls are tougher\textquotesingle{}}\NormalTok{,}
\NormalTok{       b}\StringTok{\textquotesingle{}this salad spinner does not work and I m surprised at so many positive reviews The old string pull type I had (and recently threw away alas) let you run water into an opening while you spun it then you could turn off the water and spin it dry All the way dry unlike this one Because the string spinner let you get it going fast enough to repel all the water This one won t get going fast enough to get the greens all the way dry So you have to rinse the greens separately then put them in the spinner and then finish off by drying them with a paper towel Give me back my old string spinner\textquotesingle{}}\NormalTok{,}
\NormalTok{       b}\StringTok{\textquotesingle{}i am one of paulo coelho s greatest fans when i saw this book in the bookstore couldnt think of any other thing exept that i wanted to go home and start reading after the first twenty pages i discovered that it has nothing similar to the alchamest on the contrary to me this book appeared to be very boring a waste of time it wasnt worth my exitment about it in the first place i don t advise anyone to read it it is not believable even the parts where supposedly paulo was conversing with his angel were very shallow paulo and his wife chris are too boring i felt very happy that i have finished this stupid book so that i can start another one to make make me forget my dissappointment about coelho\textquotesingle{}}\NormalTok{,}
\NormalTok{       b}\StringTok{\textquotesingle{}The first time I turned the oven on it gave off a horrible chemical smell and set off the smoke alarm Back it went to Amazon Fortunately Amazon was great about the return Ended up getting a Krups toaster oven more expensive but no major issues yet\textquotesingle{}}\NormalTok{,}
\NormalTok{       b}\StringTok{\textquotesingle{}I am an old (electrical engineer graduation ) and new engineering student (just started a bioengineering program) The new v software that came out today ( ) makes this already awesome calculator even better They have made the graphical user interface (GUI) more intuitive and have finally made the connectivity software bit compatible with Windows ( bit) I own both the TI nspires (non CAS and CAS) in order to be able to use the calculator on tests (some tests will not allow the CAS) and have to say that handheld tools have come a long way since using my HP GX for many years\textquotesingle{}}\NormalTok{,}
\NormalTok{       b}\StringTok{\textquotesingle{}Excellent product for the value Description should have listed that actual cleats were screw on Shipping was prompt Great cleats for the Weekend Athlete\textquotesingle{}}\NormalTok{,}
\NormalTok{       b}\StringTok{\textquotesingle{}I enjoyed Bryson s Walk in the Woods and recommend it frequently I wish I could say the same about this book Bryson would be the ultimate joykill on a road trip arrogant exceedingly negative and critical and unable to poke fun at himself Buy Walk in the Woods instead\textquotesingle{}}\NormalTok{,}
\NormalTok{       b}\StringTok{\textquotesingle{}I am not one to stuff into a import cd so I was pretty happy to find a Maaya Sakanoto cd that was reasonbly priced but also has almost all of my favorite tunes from her The only thing I have against this is the album art While I like the cover the album art gets pretty ridicudlous and poppy I understand well that s what J pop is like and the whole cultural thing However it still annoyed me Great cd though\textquotesingle{}}\NormalTok{,}
\NormalTok{       b}\StringTok{\textquotesingle{}This book is so far out of date as to be unusable Has very little relationship with C Builder Complete wast of money\textquotesingle{}}\NormalTok{,}
\NormalTok{       b}\StringTok{\textquotesingle{}This is not your Blackmore s Purple Tommy Bolin brings a real spark of life to the beast and it shows here s a band having FUN This is by far the most adventurous most creative and in my humble opinion best album Purple ever did Now don t get me wrong I am a fan of Purple and Blackmore but the band never sounded like they enjoyed themselves so much You can hear it listen closely this band lineup is tight All of this is very noticable if you listen to Ian s drumming he s flat out swinging Glenn s vocals and bass is right on the mark Tommy is nothing short of incredible and no he doesn t even try to do Richie he s his own man Thank God Lord and Coverdale are just like you would expect them great If your a biased Purple fan then you might not like this album but if you love great MUSIC then this one s for you\textquotesingle{}}\NormalTok{,}
\NormalTok{       b}\StringTok{\textquotesingle{}If you have read evene a few pages of any book by thix Nixonite then you hve tapped into the best that he has to offer (not much )Pass on this one\textquotesingle{}}\NormalTok{,}
\NormalTok{       b}\StringTok{\textquotesingle{}I bought the Scosche IPNRFC Wireless Remote mainly to use on my Jetski It makes it so much easier selecting artists songs creating playlists and changing the volume among other neat things I can keep my iPod securely in my glove box still have total control over everything that I want to listen to\textquotesingle{}}\NormalTok{,}
\NormalTok{       b}\StringTok{\textquotesingle{}The primary issue with his unit is that the word thermal is misleading This unit will not heat the air contrary to what you may think The bubbles will cool down the bath water\textquotesingle{}}\NormalTok{,}
\NormalTok{       b}\StringTok{\textquotesingle{}Berry reaches for the literary high ground but he stumbles on endlessly repeated imagery and numerous stunning errors of fact He s obviously interested in his topic and perhaps a less confident writer would have paid closer attention and gotten the details right\textquotesingle{}}\NormalTok{],}
      \AttributeTok{dtype=}\NormalTok{object)}\SpecialCharTok{\textgreater{}}\NormalTok{)])}

\NormalTok{[[}\DecValTok{2}\NormalTok{]]}
\FunctionTok{tf.Tensor}\NormalTok{([}\DecValTok{0} \DecValTok{0} \DecValTok{1} \DecValTok{0} \DecValTok{0} \DecValTok{1} \DecValTok{1} \DecValTok{0} \DecValTok{1} \DecValTok{0} \DecValTok{0} \DecValTok{1} \DecValTok{1} \DecValTok{1} \DecValTok{1} \DecValTok{0} \DecValTok{0} \DecValTok{1} \DecValTok{0} \DecValTok{0} \DecValTok{0} \DecValTok{0} \DecValTok{1} \DecValTok{1} \DecValTok{0} \DecValTok{1} \DecValTok{0} \DecValTok{1} \DecValTok{0} \DecValTok{1} \DecValTok{1} \DecValTok{0}\NormalTok{], }\AttributeTok{shape=}\NormalTok{(}\DecValTok{32}\NormalTok{,), }\AttributeTok{dtype=}\NormalTok{int32)}
\CommentTok{\# reticulate::py\_to\_r()}

\CommentTok{\# {-}{-}{-}{-}{-}{-}{-}{-}{-}{-}{-}{-}{-}{-}{-}{-}{-}{-}{-}{-}{-}{-}}

\NormalTok{val\_file\_path }\OtherTok{\textless{}{-}} \FunctionTok{file.path}\NormalTok{(}\StringTok{"amazon\_review\_polarity\_csv/amazon\_val.csv"}\NormalTok{)}

\NormalTok{val\_dataset }\OtherTok{\textless{}{-}} \FunctionTok{make\_csv\_dataset}\NormalTok{(val\_file\_path, }\AttributeTok{field\_delim =} \StringTok{","}\NormalTok{, }\AttributeTok{batch\_size =}\NormalTok{ batch\_size,}
    \AttributeTok{column\_names =} \FunctionTok{list}\NormalTok{(}\StringTok{"label"}\NormalTok{, }\StringTok{"title"}\NormalTok{, }\StringTok{"text"}\NormalTok{), }\AttributeTok{label\_name =} \StringTok{"label"}\NormalTok{, }\AttributeTok{select\_columns =} \FunctionTok{list}\NormalTok{(}\StringTok{"label"}\NormalTok{,}
        \StringTok{"text"}\NormalTok{), }\AttributeTok{num\_epochs =} \DecValTok{1}\NormalTok{)}


\NormalTok{val\_dataset }\SpecialCharTok{\%\textgreater{}\%}
\NormalTok{    reticulate}\SpecialCharTok{::}\FunctionTok{as\_iterator}\NormalTok{() }\SpecialCharTok{\%\textgreater{}\%}
\NormalTok{    reticulate}\SpecialCharTok{::}\FunctionTok{iter\_next}\NormalTok{()}
\NormalTok{[[}\DecValTok{1}\NormalTok{]]}
\FunctionTok{OrderedDict}\NormalTok{([(}\StringTok{\textquotesingle{}text\textquotesingle{}}\NormalTok{, }\SpecialCharTok{\textless{}}\NormalTok{tf.Tensor}\SpecialCharTok{:} \AttributeTok{shape=}\NormalTok{(}\DecValTok{32}\NormalTok{,), }\AttributeTok{dtype=}\NormalTok{string, }\AttributeTok{numpy=}
\FunctionTok{array}\NormalTok{([b}\StringTok{\textquotesingle{}Wish I d caught on earlier Everyone told me that the Harry Potter series was good I my expectations were definitely exceeded\textquotesingle{}}\NormalTok{,}
\NormalTok{       b}\StringTok{\textquotesingle{}Jennifer Love Hewitt doesn t look a thing like Audrey Hepburn I truely believe that the producers and casting directors made a terrible call casting Hewitt The actress who I believe reminds people of Audrey as well as looks like her is Natalie Portman (who is an exceptional actress) Natalie would have made a much mcuh better Audrey Hepburn\textquotesingle{}}\NormalTok{,}
\NormalTok{       b}\StringTok{\textquotesingle{}Absolutely charming book great for all grade levels I usually run for the gory werewolfie or anne boleynesque books but it was refreshing to pick up something sweet AND interesting for a change Buy this for the MG reader in your life but read it for yourself first\textquotesingle{}}\NormalTok{,}
\NormalTok{       b}\StringTok{\textquotesingle{}This movie was made to be a fantasy martial arts movie and it delivers as promised I have a very extensive collection of Martial Arts movies and I can honestly say that this movie is one of my top favorite movies I gave it a instead of a five because it does go a little overboard with the fantasy element but not enough to distract you from the fighting and believe me there s a lot of it\textquotesingle{}}\NormalTok{,}
\NormalTok{       b}\StringTok{\textquotesingle{}Aircraft physics cgi are horrible Planes don t fly that way And the German guy is of course dressed in black and he s really bad The hero becomes obsessed with getting even and of course does Most video games are superior to this sillyness Total waste\textquotesingle{}}\NormalTok{,}
\NormalTok{       b}\StringTok{\textquotesingle{}I had never read Treasure Island but upon receiving my kindle and the fact this was a free read I downloaded a copy I had always thought I should sit and read this one and I am happy I did The story never really got stagnate As in the story hanging around the sea port town too long or sitting in the island cabin The story keeps moving along with twists and turns It was a great adventure on a great ship with the cook to a little known island If you havn t read it and enjoy adventure stories and mystery dissete and double crossing and most of all sea adventures I suggest you try out this gem It s not too long and a great read\textquotesingle{}}\NormalTok{,}
\NormalTok{       b}\StringTok{\textquotesingle{}I tend to agree with some minority reviewers that this book is boring and dull It has some interesting thoughts here and there but that is all All in all the book is quite over rated\textquotesingle{}}\NormalTok{,}
\NormalTok{       b}\StringTok{\textquotesingle{}I read about halfway through this book and then I gave up I read James Turn of the Screw and Daisy Miller in high school and I remember liking the former and thinking the latter was just okay (I know I know it s a major classic by one of America s most celebrated writers but just because something has merit doesn t mean I like it better ) One of my all time favorite books was James Washington Square It s hard for me to believe that the same man wrote Square and Maisie This book is only for MAJOR Henry James enthusiasts\textquotesingle{}}\NormalTok{,}
\NormalTok{       b}\StringTok{\textquotesingle{}Typical cavalry versus Indian oater Lots of action which is what I ask from a Western\textquotesingle{}}\NormalTok{,}
\NormalTok{       b}\StringTok{\textquotesingle{}This movie is pretty good It didn t require an indepth knowledge of the TV show to follow the movie\textquotesingle{}}\NormalTok{,}
\NormalTok{       b}\StringTok{\textquotesingle{}This is first time I have been compelled to write a bad review but frankly this movie was bad Don t get me wrong there is nothing I love more than a romantic comedy However this was not romantic and not funny I didn t expect an Academy Award winning movie just a cute entertaining romantic comedy It didn t deliver The two stars have been great in everything I have seen them in but this movie just did not work for them I felt no chemistry between them Chris O Donnell just irritated me the entire movie and the lines these actors were expected to deliver were horrible I was embarrased watching this movie I ve read the previous reviews and I must say I m interested in seeing the original so I can see what they could have done with this movie Skip this one and see Return to Me when it comes out on DVD Video\textquotesingle{}}\NormalTok{,}
\NormalTok{       b}\StringTok{\textquotesingle{}The Third Secret is another entry in the Vatican intrigue genre It has the usual elements Vatican conservatives the usual bad guys vs liberal Catholics the usual good guys The Third Secret refers to secret revelations given to Marian visionaries at such places as Fatima and Medjugorge the existence of which is historical fact If there is any suspense in the novel it is not with the plot which is remarkably predictable and derivative The only thing that kept my interest was waiting to find out the actual content of the third secret(s) at least according to the novel s author In the end these proved to be rather predictable too given the author s religious inclinations\textquotesingle{}}\NormalTok{,}
\NormalTok{       b}\StringTok{\textquotesingle{}I can t believe I wasted my money on this When I got the product I had before hand read some reviews about it falling apart or something So before I wasted the sand that came with it and caused a mess I gently pressed on the wood base part It was flimsy but I didn t think much of it Then while just doing a quick look over I noticed that there was a small disconect between the bottom base and the black lining This would cause the sand to leak out of the crack Then the base fell write off I noticed that it had very tiny small metal pieces that had wood from the base around it I now realize that they probably put the metal into it painted the top black then pushed on the base to connect the two parts The little stubs that hold it up have superglue stains from where they glued it to the bottom of the base How cheap I really don t know what to do now I ll probably get my dad to put some nails in it Really the Dollar Tree would probably make a better structure\textquotesingle{}}\NormalTok{,}
\NormalTok{       b}\StringTok{\textquotesingle{}The Spongebob Squarepants Movie had an unoriginal title but I LOVED the movie anyway I only liked two songs from it tho Goofy Goober Rock and The Best Day Ever But overall I recommend it for kids yrs old to adult This movie isn t a waste of time or money so buy it now I also recommend this for anyone who loves Spongebob (and Patrick) who loves silly but sometimes twisted comedy or someone who just wants to have a good time\textquotesingle{}}\NormalTok{,}
\NormalTok{       b}\StringTok{\textquotesingle{}Under the table surface there are two factory assembled bars used to connect the four legs Clearly the bars on my table were misaligned as they almost stick out of the table surface at one end (I will upload a photo for that) and left no room for the legs This rendered the whole table useless It surprised me how such a defected part could get through the manufacturer s quality control You don t need any fancy tool to discover it Looking at it would suffice Finally I chose to fix it myself It was not very difficult for me just taking off the screws that hold the bars realigning the bars to the correct position and then put the screws back in again The extra minutes work was still worth it comparing to repackaging the whole set the hassle to ask for a return plus shopping for another table The quality of the table (other than the misaligned bars) is OK That s why I still give it two stars But I would really hesitate to buy again from this manufacturer\textquotesingle{}}\NormalTok{,}
\NormalTok{       b}\StringTok{\textquotesingle{}I read the comment by the reviewer from Montgomery Alabama back in September that the ASV would be available from Star Bible in November I can find no information about Star Bible and the ASV still appears as out of stock or out of print in all my searches on Amazon Can anyone help me find a source for a printed copy of the ASV If so please respond here or email to nathancci hotmail com Thank you very much\textquotesingle{}}\NormalTok{,}
\NormalTok{       b}\StringTok{\textquotesingle{}I ordered these beds because my grandkids were sleeping on the floor and couldnt afford real beds and in less than months both beds have popped They didnt jump on them they just lost air seemed like the seams were the problem bed was replaced but the other remains not replaced I would never buy these type of beds ever again or this brand They were not cheap and I cant afford to replace them again So now they are back to sleeping on the floor again I heard they were good or I would not have ordered them I am terribly disappointed\textquotesingle{}}\NormalTok{,}
\NormalTok{       b}\StringTok{\textquotesingle{}I looked forward to seeing this movie but it was a huge letdown Besides the historical inaccuracies as detailed by earlier reviewers here on Amazon I felt the actors chosen to play these music icons weren t based on ability but to try and get a younger audience to pay attention to the film Mos Def is NOT an actor in any way shape or form his portrayal of Chuck Berry was wooden and inaccurate On the other hand Eamonn Walker s take on Howlin Wolf was STUNNING I m sure the makers intentions were good but the obvious budget restraints and poor choices in actors left me feeling like a great opportunity to shed light on an otherwise under documented period of music history was squandered\textquotesingle{}}\NormalTok{,}
\NormalTok{       b}\StringTok{\textquotesingle{}Bought this book by accident and now I love it It is easy to take to clinicals or even to look up different disease process it has nsg diagnosis with each one also and it is cheap\textquotesingle{}}\NormalTok{,}
\NormalTok{       b}\StringTok{\textquotesingle{}I went into this movie that it was going to be stupid But I didn t even find it funny Sure there are some scenes that make you laugh but its cause they are so pathetic The acting was terrible all around and the story made no sense what so ever This is just a bad movie with a cool title\textquotesingle{}}\NormalTok{,}
\NormalTok{       b}\StringTok{\textquotesingle{}And these masters are Michael Bay and his computers And we have no one but ourselves to blame This movie is like a hours car crash When you think it can t get any worse it does It would deserve stars if it didn t have any human actors But Michael Bay is actually controlled by his computers and they made him put these humans in this movie to make us suffer even more This is all I have to say Transformers IV is coming soon to the theaters near you Preorder your tickets now\textquotesingle{}}\NormalTok{,}
\NormalTok{       b}\StringTok{\textquotesingle{}I was surprised and delighted to see this remake After purchasing I was well disappointed I guess because the framework of the story was the same but nothing else was The modernisation of the action was poor the story only Ok and I didn t find it engaging except in one scene A satisfying ending with all the story nicely wrapped up but I think the cast deserved a better story line and sticking to the original concept and doing a real remake with modern cinema techniques could have made this a four star candidate Oh well I guess we ll never see a good modern updated version because this certainly is not it\textquotesingle{}}\NormalTok{,}
\NormalTok{       b}\StringTok{\textquotesingle{}The whole album is amazing and hyper ballad has got to be one of the most beautiful songs ever\textquotesingle{}}\NormalTok{,}
\NormalTok{       b}\StringTok{\textquotesingle{}I have a Samsung Blu Ray and the BR disc of the th Anniversary version of Dirty Dancing The picture quality of the movie is just awful particularly on the indoor scenes Faces and backgrounds were almost fuchsia almost always I did not adjust my TV color since all of the extras including extended scenes outtakes and deleted scenes were perfect quality What a pity My first BR disc purchase and such a disappointment The manufacturer should be ashamed Dirty Dancing ( th Anniversary Edition) Blu ray\textquotesingle{}}\NormalTok{,}
\NormalTok{       b}\StringTok{\textquotesingle{}I like family love movie very much From the view point of a father this movie is excellent In the movie Frank saved his son John who was just murdered at the last scene I was moved by the great love of a father Probably I wouldn t show my love to my sons as much as Frank In Japan mothers usually take care of children and fathers aren t good at showing their love to family members So a father tends to be isolated from his family Of course I am always concerned about my family But I don t tell them my concerns directly So my affections aren t handed to them I think Frank is a father of fathers I try to imitate Frank The movie tells me an important role of a father\textquotesingle{}}\NormalTok{,}
\NormalTok{       b}\StringTok{\textquotesingle{}Mine quite pumping water after cups I should have known better since my oldest son had sent his back times before getting one that worked Now his is once again screwing up I thought that for the amount of time this model had been on the market it would have been fixed I have an older B we have been using for a long time but it is screwing up now That s why I ordered this one The B won t fit under our cabinets or I would have tried it No more Keurig s for me I will try another brand now\textquotesingle{}}\NormalTok{,}
\NormalTok{       b}\StringTok{\textquotesingle{}I am the author of this book It is no longer being printed I gave it only four stars because its successor is deserving of five stars Go to the following Amazon com page to see West Side Publishing s Armchair Reader The Last Survivors of Historical Events Movies Disasters and More Armchair Reader The Last Survivors of Historical Events Movies Disasters and MoreRobert Ernest Hubbard\textquotesingle{}}\NormalTok{,}
\NormalTok{       b}\StringTok{\textquotesingle{}This is the book that got me interested in reading mystery novels It s suspenseful thought provoking and totally unpredictable I have yet to find another mystery that really adds up to what Doyle has created in these few pages I want to read all his novels now and I can t wait\textquotesingle{}}\NormalTok{,}
\NormalTok{       b}\StringTok{\textquotesingle{}I did not really care for this film though it did have a few funny moments a lot of the attempts at humor fell flat for me I really like Martin Short but didn t think he was very funny in this It was a little corny I also thought that it was going to be more of a musical but there was very little singing so I was disappointed in that I also didn t like the moral behind the plot of the wish itself getting the little girls dad the role in the play by using any means to sabbotage the character who had the role I don t think that s a good message for kids even if the methods used are done by fairy godmothers I know a lot of shows do that like Bruce Almighty but in even that show in the end he gave the newscaster s job back to the character that he stole it from I wouldn t recommend this movie to anyone\textquotesingle{}}\NormalTok{,}
\NormalTok{       b}\StringTok{\textquotesingle{}This book had so much potential to be far more than what it was I agree with other customers who said it started off well but all of sudden there were so many U turns and loops and twists that it seems as though the writers were still brainstorming what they wanted to do with all the information they had It was poorly written and it takes away what probably could have been a journey worth reading I think maybe Lula should have considered someone else to do her biography Either these writers were inexperienced or had too many projects on their plates and just stuffed this one in\textquotesingle{}}\NormalTok{,}
\NormalTok{       b}\StringTok{\textquotesingle{}I wear a size I took the info left by others and ordered a size They fit great Warm but not hot and the this rubber sole means I can let the dogs out without having to take them off\textquotesingle{}}\NormalTok{,}
\NormalTok{       b}\StringTok{\textquotesingle{}Mr Waltari s storytelling style comes forth in colorful detail and complex characters The mix of history intrigue and involves the reader in a way that one feels as if one is in these ancient places with these people The characters are fleshed out to the point that one could almost toch these people and engage them in a conversation The Egyptian is a tour de force of storyteling and social commentary It is a great read and a compelling story for lovers of history and epics\textquotesingle{}}\NormalTok{],}
      \AttributeTok{dtype=}\NormalTok{object)}\SpecialCharTok{\textgreater{}}\NormalTok{)])}

\NormalTok{[[}\DecValTok{2}\NormalTok{]]}
\FunctionTok{tf.Tensor}\NormalTok{([}\DecValTok{1} \DecValTok{0} \DecValTok{1} \DecValTok{1} \DecValTok{0} \DecValTok{1} \DecValTok{0} \DecValTok{0} \DecValTok{1} \DecValTok{1} \DecValTok{0} \DecValTok{0} \DecValTok{0} \DecValTok{1} \DecValTok{0} \DecValTok{1} \DecValTok{0} \DecValTok{0} \DecValTok{1} \DecValTok{0} \DecValTok{0} \DecValTok{0} \DecValTok{1} \DecValTok{0} \DecValTok{1} \DecValTok{0} \DecValTok{1} \DecValTok{1} \DecValTok{0} \DecValTok{0} \DecValTok{1} \DecValTok{1}\NormalTok{], }\AttributeTok{shape=}\NormalTok{(}\DecValTok{32}\NormalTok{,), }\AttributeTok{dtype=}\NormalTok{int32)}

\CommentTok{\# {-}{-}{-}{-}{-}{-}{-}{-}{-}{-}{-}{-}{-}{-}{-}{-}{-}{-}{-}{-}{-}{-}{-}{-}{-}{-}{-}{-}{-}{-}{-}{-}{-}{-}{-}}

\NormalTok{test\_file\_path }\OtherTok{\textless{}{-}} \FunctionTok{file.path}\NormalTok{(}\StringTok{"amazon\_review\_polarity\_csv/amazon\_test.csv"}\NormalTok{)}


\NormalTok{test\_dataset }\OtherTok{\textless{}{-}} \FunctionTok{make\_csv\_dataset}\NormalTok{(test\_file\_path, }\AttributeTok{field\_delim =} \StringTok{","}\NormalTok{, }\AttributeTok{batch\_size =}\NormalTok{ batch\_size,}
    \AttributeTok{column\_names =} \FunctionTok{list}\NormalTok{(}\StringTok{"label"}\NormalTok{, }\StringTok{"title"}\NormalTok{, }\StringTok{"text"}\NormalTok{), }\AttributeTok{label\_name =} \StringTok{"label"}\NormalTok{, }\AttributeTok{select\_columns =} \FunctionTok{list}\NormalTok{(}\StringTok{"label"}\NormalTok{,}
        \StringTok{"text"}\NormalTok{), }\AttributeTok{num\_epochs =} \DecValTok{1}\NormalTok{)}

\NormalTok{test\_dataset }\SpecialCharTok{\%\textgreater{}\%}
\NormalTok{    reticulate}\SpecialCharTok{::}\FunctionTok{as\_iterator}\NormalTok{() }\SpecialCharTok{\%\textgreater{}\%}
\NormalTok{    reticulate}\SpecialCharTok{::}\FunctionTok{iter\_next}\NormalTok{()}
\NormalTok{[[}\DecValTok{1}\NormalTok{]]}
\FunctionTok{OrderedDict}\NormalTok{([(}\StringTok{\textquotesingle{}text\textquotesingle{}}\NormalTok{, }\SpecialCharTok{\textless{}}\NormalTok{tf.Tensor}\SpecialCharTok{:} \AttributeTok{shape=}\NormalTok{(}\DecValTok{32}\NormalTok{,), }\AttributeTok{dtype=}\NormalTok{string, }\AttributeTok{numpy=}
\FunctionTok{array}\NormalTok{([b}\StringTok{\textquotesingle{}The one star is for Liam s hair on the inside cover At least that still has it s dignity As for the music I believe they have finally lost their touch The Importance of Being Idle Part of the Queue and Let There Be Love are mediocre Unfortunately all of the other tracks are completely unlistenable It s sad that a band who was once so great are putting out this kind of trash but luckily for their fans they let us down slowly with Standing On The Shoulders Of Giants and Heathen Chemistry so at least it didn t come as a big surprise\textquotesingle{}}\NormalTok{,}
\NormalTok{       b}\StringTok{\textquotesingle{}The spreader is much larger and wider than I wanted I have found it useless for buttering bread sticks which is why it was ordered This is my fault and not that of the spreader\textquotesingle{}}\NormalTok{,}
\NormalTok{       b}\StringTok{\textquotesingle{}This is so typical of H James writing very make that incredibly dense Wordy beyond belief At a pace that surely puts me fast asleep after a mere pages A ridiculous pair of uncaring parents essentially abandon their daughter to whomever comes along As an amazing coincidence the divorced parents each take a new spouse only to watch the wife of one have an affair with the husband of the other If you are really into dialogue that no one has ever spoken in the history of English and willing to devote hours to really obscure language construction this is the very winner for you\textquotesingle{}}\NormalTok{,}
\NormalTok{       b}\StringTok{\textquotesingle{}I previous had a similar jar opener and have places and wanted to keep one in one place and the other in another This jar opener is an inferior product and the worse thing I ever purchased from Amazon(and I purchased numerous things)It doesn t grips all jars and it should have advised me of what it was made of It looked various similar to the other one I own and I can t wait till I can find another jar opener like the other one so I can throw this one in the garbage\textquotesingle{}}\NormalTok{,}
\NormalTok{       b}\StringTok{\textquotesingle{}Wasn t sure what was wrong with my ice maker so I bought this kit Replaced just the main ice maker component (took less than minutes) and the unit started right up It has been making ice reliably Great deal\textquotesingle{}}\NormalTok{,}
\NormalTok{       b}\StringTok{\textquotesingle{}This book is a watershed in human intellectual history In it Freud undermines the picture of mankind as primarily a being of reason and presents the idea that we are all creatures of our wishes our inner unconscious lives Dreams are not nothing and they are not in Freud s eyes rare religious gifts but rather to the key to our own mental life Freud in this book presents a vast world of examples and interpretations I am not a psychologist and do not consider myself competent to really judge how much of what Freud presents here is valid or even capable of scientific testing I do know that this work is one which like a great literary masterpiece has inspired countless interpretations and reinterpretations Understanding human Intellectual History is now impossible without knowing this work\textquotesingle{}}\NormalTok{,}
\NormalTok{       b}\StringTok{\textquotesingle{}The game includes nowhere to store the black sheets for the screen The storage drawer jams every time you close it so opening it tends to result in pegs flying all over the room You can turn the light on but not off I seem to recall that we had on off light technology even way back when I was a kid The pegs don t stay in well the blue pegs tend not to glow at all unless they re directly above the light bulb You d think with many generations of children to evolve and perfect this toy it would have gotten better rather than worse your expectations would be frustrated\textquotesingle{}}\NormalTok{,}
\NormalTok{       b}\StringTok{\textquotesingle{}This is a very good cd you should buy it My favourite songs are Why does my heart feel so bad Porcelain guitar flute and stings and although almost all songs on the album are good\textquotesingle{}}\NormalTok{,}
\NormalTok{       b}\StringTok{\textquotesingle{}I thought at first that the vacuum was extremely hard to push until I realized that the lowest setting was for smooth floors rather than carpet Once I adjusted the dial the vacuum moved much more easily It has great suction and picks up so much dirt and dog hair from my carpet that I m embarrassed about my housekeeping I really like being able to see in the clear cylinder what it s picking up from the floor and it is very easy to empty the canister The cord rewinder is excellent The vacuum s a bit loud but no more than most that I ve had It s definitely worth the money\textquotesingle{}}\NormalTok{,}
\NormalTok{       b}\StringTok{\textquotesingle{}There are many accolades and volumes of in depth analysis for this film so my review will be short The good Strong weak sad wise frail crazy calm etc the full gamut of characters that really work together to twist complex situations The story keeps you guessing first time It is timeless and the subitles did not bother me because the pace of the movie is easy enough to keep up with reading but also watching The bad It can move too slow at times and I had to break up the film and view over two nights Conclusion This is a movie I will watch again probably a few times if lucky\textquotesingle{}}\NormalTok{,}
\NormalTok{       b}\StringTok{\textquotesingle{}This book did not move me at all The plot is unrealistic and the characters are shallow they do not seem like real people at all\textquotesingle{}}\NormalTok{,}
\NormalTok{       b}\StringTok{\textquotesingle{}Aaliyah has definitely grown up The songs are about love hurt pain sex etc The songs are more sensual than her usual style My favorite track is Rock the boat then I care for you then I Refuse I am a huge Aaliyah fan Keep up the good work baby girl\textquotesingle{}}\NormalTok{,}
\NormalTok{       b}\StringTok{\textquotesingle{}Whoa boy I ve seen some pretty lame flicks last year but The Skulls definitely comes in as one of the worst It s a mind bogglingly stupid tale about a pretty boy (Joshua Jackson) who stupidly decides to join a secret society and it seems all the members of this society called The Skulls are pretty dumb in their own rights too Of course this didn t prepare me for the lame as heck finale which features a duel that s right a duel involving two flintlock pistols\textquotesingle{}}\NormalTok{,}
\NormalTok{       b}\StringTok{\textquotesingle{}In Flames and Soilwork are sold out so my only chance to find another album that blow my mind like Dark Tranqility s Character is this This album is solid musicianship throghout There s no filler here or the commercial flavor like the other two(In Flames Soilwork) this is great melodic death metal with no complaints There s industrial touches like in Character but sounds really good Check this out you won t regret it\textquotesingle{}}\NormalTok{,}
\NormalTok{       b}\StringTok{\textquotesingle{}I guess I should preface this by saying that I m really not an Asian movie person I just don t get them and I m sure it s a cultural thing Tampopo Crouching Tiger In the Realm of the Senses and now this one I couldn t sit through any of them Unless you are a martial arts or samurai fanatic or a big lover of Japanese culture and history I would say definitely skip this especially if you are female I think this is one of those Emperor s New Movies that we re all supposed to like and no one will admit to NOT liking it for fear of being thought ignorant\textquotesingle{}}\NormalTok{,}
\NormalTok{       b}\StringTok{\textquotesingle{}OPENED THE BOX AND IT WAS NOT CARBON FIBER IT WAS CLEAR AMAZON NEEDS TO CHECK THEIR INVENTORY ON THIS update NOV Amazon s replacement was still the same it s not d carbon fiber will return for a refund\textquotesingle{}}\NormalTok{,}
\NormalTok{       b}\StringTok{\textquotesingle{}Danger Kitty has mass produced some of the worst music in the history of Pop To all of the BJ fans out there look up the word cliche Understand its danger and the shallow nature that this word connotes It s ahem like a wet slippery loaded gun If you re going to write lyrics this horrible juvenile and unimaginative why bother finishing Junior High at all Drop out of school get yourself a job at Hot Dog on a Stick and say goodbye to every one of your brain cells Patriotism demands that you rid your CD player of anything that makes you look like you couldn t finish the th grade Do it for your country\textquotesingle{}}\NormalTok{,}
\NormalTok{       b}\StringTok{\textquotesingle{}I am only years old but at least read a book a week When i first picked up this book i felt i would not beable to finish it because of my ablity to get bored really quick I started reading it and couldn t put it down I got in trouble in classes because i had been reading it under my desk until i reached a point (which was very rare) that i could put it down I love Jane Eyre s way of thinking and how she trys to make her life as full as possible I would recomend this to everyone I keep on going back in the book to a part i liked best or a place in the story that captivates me It seems i will never grow tired of this book I hope all that have read it feel the same\textquotesingle{}}\NormalTok{,}
\NormalTok{       b}\StringTok{\textquotesingle{}I thought the book was very good It does get a little boring to me because of so much explicit detail written in on everything The book is around pages but well worth the read Can t wait for the final book\textquotesingle{}}\NormalTok{,}
\NormalTok{       b}\StringTok{\textquotesingle{}This book was given to me I read it and placed it on my book shelf As a writer I have used the quotes in articles and letters A perfect addition to your library if you write to or about children\textquotesingle{}}\NormalTok{,}
\NormalTok{       b}\StringTok{\textquotesingle{}The first of this movie are spent developing the characters of Liu Xing and Prof Reiser and then all the strongest character traits of those two people are just tossed aside for the ridiculous ending That made the entire film a complete waste of time You re left wondering what moron wrote that idiotic ending and why Anyone with a shred of intellect isn t going to be moved by this except maybe to anger at having squandered minutes on it It could have been a decent movie It did have a really cheap quality about it as you never see ANY other students at the school\textquotesingle{}}\NormalTok{,}
\NormalTok{       b}\StringTok{\textquotesingle{}The Nantucket Blend is one of my favorite k cup coffees that I ve made It s a medium blend coffee that tastes great with flavored creamer or just regular creamer or milk I will definitely buy more of this one\textquotesingle{}}\NormalTok{,}
\NormalTok{       b}\StringTok{\textquotesingle{}My wife and I are planning a trip to Tuscany in May and ordered this video to get ideas about off the beaten track places to go The video is beautifully shot well narrated and contains useful information On the negative side though its a bit thin on detail and far too short in running time It left us wanting more but certainly excited about our upcoming trip In that sense I suppose it was worth the money\textquotesingle{}}\NormalTok{,}
\NormalTok{       b}\StringTok{\textquotesingle{}Okay if you re a special effects freak you might like this However there are much better films with equally good effects If you re a fan of minor things like say oh plot character development etc then stay away The Robert Wise film is the scariest film I ve ever seen and the scares come from what Wise DOESN T show you The version was brilliantly plotted and paced with one of the greatest ending lines in film history The version has none of this Do yourself a favor and see the movie or better yet read the Shirley Jackson novel The only reason this gets one star is because C Zeta Jones is just SO easy on the eyes\textquotesingle{}}\NormalTok{,}
\NormalTok{       b}\StringTok{\textquotesingle{}I checked this book out from my local La Leche League I m surprised they still have this book available Being nearly twenty years old maybe some of my problems are with it being outdated However my main gripe is that it is very condescending and judgmental Instead of just informing you on different options (none of which would work for me and only one or so that would apply well to a full time job) this book presents a certain ideal and if the job arrangements do not fit that ideal they re not good arrangements and you aren t a good mother If you want helpful non opinionated advice this is not the book for you Currently I m reading The Working Mother s Guide to Life which seems much more friendly and helpful\textquotesingle{}}\NormalTok{,}
\NormalTok{       b}\StringTok{\textquotesingle{}Bought this for wife as a stocking stuffer I m sure she ll enjoy backing her team without taking up the whole window\textquotesingle{}}\NormalTok{,}
\NormalTok{       b}\StringTok{\textquotesingle{}Recently bought the Sony SS B speakers As many have mentioned are a little larger than most bookshelf speakers However these speakers are an exceptional value for the money Plus they sound fantastic clear highs smooth mid range and good bass response Most music lovers will be satisfied with the sound and these are also nice looking speakers You cannot go wrong\textquotesingle{}}\NormalTok{,}
\NormalTok{       b}\StringTok{\textquotesingle{}Was Very Very disappointed Movie though brand new just out of the box it was stopping all the time during the latter part of the movie\textquotesingle{}}\NormalTok{,}
\NormalTok{       b}\StringTok{\textquotesingle{}This is the worst movie you will ever watch no plot no story terrible casting and nothing to do with the original series Ugh It sucks\textquotesingle{}}\NormalTok{,}
\NormalTok{       b}\StringTok{\textquotesingle{}I just got this unit and must say that I am disappointed It maybe because I was expecting it fully decode hdmi audio instead of having another optical in for the same source So when I connect my PS I can not listen to full uncompressed audio in LPCM which is available only in hdmi If you have only optical audio to take care of you will be find with this receiver But if you are looking at it just because it has HDMI beware and read the fine prints\textquotesingle{}}\NormalTok{,}
\NormalTok{       b}\StringTok{\textquotesingle{}This book is stupid the romance the vampires everything and why did she use that poem Amelia never actually stated if these vampires are demon or not They hold emotions of love and that doesn t make em demon\textquotesingle{}}\NormalTok{,}
\NormalTok{       b}\StringTok{\textquotesingle{}My Xantrex XPower Powerpack Heavy Duty is not good for an emergency It is an emergency The unit does not hold a charge It takes days to fully charge and loses its charge by each day Xantrex customer service has not been helpful other than to say that I should charge it through my car cigarette lighter I ask what should I do charge it all the time Thats the only way their will be any power when you need it I bought this item to help if I have a problem It is the problem\textquotesingle{}}\NormalTok{],}
      \AttributeTok{dtype=}\NormalTok{object)}\SpecialCharTok{\textgreater{}}\NormalTok{)])}

\NormalTok{[[}\DecValTok{2}\NormalTok{]]}
\FunctionTok{tf.Tensor}\NormalTok{([}\DecValTok{0} \DecValTok{0} \DecValTok{0} \DecValTok{0} \DecValTok{1} \DecValTok{1} \DecValTok{0} \DecValTok{1} \DecValTok{1} \DecValTok{1} \DecValTok{0} \DecValTok{1} \DecValTok{0} \DecValTok{1} \DecValTok{0} \DecValTok{0} \DecValTok{0} \DecValTok{1} \DecValTok{1} \DecValTok{1} \DecValTok{0} \DecValTok{1} \DecValTok{1} \DecValTok{0} \DecValTok{0} \DecValTok{1} \DecValTok{1} \DecValTok{0} \DecValTok{0} \DecValTok{0} \DecValTok{0} \DecValTok{0}\NormalTok{], }\AttributeTok{shape=}\NormalTok{(}\DecValTok{32}\NormalTok{,), }\AttributeTok{dtype=}\NormalTok{int32)}

\FunctionTok{rm}\NormalTok{(amazon\_orig\_train, amazon\_orig\_test, amazon\_train, amazon\_val)}
\NormalTok{Warning }\ControlFlowTok{in} \FunctionTok{rm}\NormalTok{(amazon\_orig\_train, amazon\_orig\_test, amazon\_train, amazon\_val)}\SpecialCharTok{:}
\NormalTok{object }\StringTok{\textquotesingle{}amazon\_orig\_train\textquotesingle{}}\NormalTok{ not found}
\NormalTok{Warning }\ControlFlowTok{in} \FunctionTok{rm}\NormalTok{(amazon\_orig\_train, amazon\_orig\_test, amazon\_train, amazon\_val)}\SpecialCharTok{:}
\NormalTok{object }\StringTok{\textquotesingle{}amazon\_orig\_test\textquotesingle{}}\NormalTok{ not found}
\NormalTok{Warning }\ControlFlowTok{in} \FunctionTok{rm}\NormalTok{(amazon\_orig\_train, amazon\_orig\_test, amazon\_train, amazon\_val)}\SpecialCharTok{:}
\NormalTok{object }\StringTok{\textquotesingle{}amazon\_val\textquotesingle{}}\NormalTok{ not found}
\FunctionTok{rm}\NormalTok{(ids\_train, train\_file\_path, test\_file\_path, val\_file\_path)}
\NormalTok{Warning }\ControlFlowTok{in} \FunctionTok{rm}\NormalTok{(ids\_train, train\_file\_path, test\_file\_path, val\_file\_path)}\SpecialCharTok{:}\NormalTok{ object}
\StringTok{\textquotesingle{}ids\_train\textquotesingle{}}\NormalTok{ not found}
\end{Highlighting}
\end{Shaded}

\newpage

\hypertarget{the-preprocessing-model}{%
\subsection{The preprocessing model}\label{the-preprocessing-model}}

Text inputs need to be transformed to numeric token ids and arranged in
several Tensors before being input to BERT. TensorFlow Hub provides a
matching preprocessing model for the BERT models, which implements this
transformation using TF ops from the TF.text library.

The preprocessing model must be the one referenced by the documentation
of the BERT model, which you can read at the URL
\href{https://tfhub.dev/tensorflow/bert_en_uncased_preprocess/3}{bert\_en\_uncased\_preprocess}

\begin{Shaded}
\begin{Highlighting}[]
\NormalTok{bert\_preprocess\_model }\OtherTok{\textless{}{-}} \FunctionTok{layer\_hub}\NormalTok{(}\AttributeTok{handle =} \StringTok{"https://tfhub.dev/tensorflow/bert\_en\_uncased\_preprocess/3"}\NormalTok{,}
    \AttributeTok{trainable =} \ConstantTok{FALSE}\NormalTok{, }\AttributeTok{name =} \StringTok{"preprocessing"}\NormalTok{)}
\end{Highlighting}
\end{Shaded}

\hypertarget{using-the-bert-model}{%
\subsection{Using the BERT model}\label{using-the-bert-model}}

The BERT models return a map with 3 important keys: pooled\_output,
sequence\_output, encoder\_outputs:

``pooled\_output'' represents each input sequence as a whole. The shape
is {[}batch\_size, H{]}. You can think of this as an embedding for the
entire Amazon review.

``sequence\_output'' represents each input token in the context. The
shape is {[}batch\_size, seq\_length, H{]}. You can think of this as a
contextual embedding for every token in the Amazon review.

``encoder\_outputs'' are the intermediate activations of the L
Transformer blocks. outputs{[}``encoder\_outputs''{]}{[}i{]} is a Tensor
of shape {[}batch\_size, seq\_length, 1024{]} with the outputs of the
i-th Transformer block, for 0 \textless= i \textless{} L. The last value
of the list is equal to sequence\_output.

For the fine-tuning you are going to use the pooled\_output array.

For more information about the base model's input and output you can
follow the model's URL at
\href{https://tfhub.dev/tensorflow/small_bert/bert_en_uncased_L-4_H-512_A-8/2}{small\_bert/bert\_en\_uncased\_L-4\_H-512\_A-8}

\begin{Shaded}
\begin{Highlighting}[]
\NormalTok{bert\_model }\OtherTok{\textless{}{-}} \FunctionTok{layer\_hub}\NormalTok{(}\AttributeTok{handle =} \StringTok{"https://tfhub.dev/tensorflow/small\_bert/bert\_en\_uncased\_L{-}4\_H{-}512\_A{-}8/2"}\NormalTok{,}
    \AttributeTok{trainable =} \ConstantTok{TRUE}\NormalTok{, }\AttributeTok{name =} \StringTok{"BERT\_encoder"}\NormalTok{)}
\end{Highlighting}
\end{Shaded}

\hypertarget{define-your-model}{%
\subsection{Define your model}\label{define-your-model}}

We will create a very simple fine-tuned model, with the preprocessing
model, the selected BERT model, one Dense and a Dropout layer.

\begin{Shaded}
\begin{Highlighting}[]
\NormalTok{input }\OtherTok{\textless{}{-}} \FunctionTok{layer\_input}\NormalTok{(}\AttributeTok{shape =} \FunctionTok{shape}\NormalTok{(), }\AttributeTok{dtype =} \StringTok{"string"}\NormalTok{, }\AttributeTok{name =} \StringTok{"text"}\NormalTok{)}

\NormalTok{output }\OtherTok{\textless{}{-}}\NormalTok{ input }\SpecialCharTok{\%\textgreater{}\%}
    \FunctionTok{bert\_preprocess\_model}\NormalTok{() }\SpecialCharTok{\%\textgreater{}\%}
\NormalTok{    bert\_model }\SpecialCharTok{\%$\%}
\NormalTok{    pooled\_output }\SpecialCharTok{\%\textgreater{}\%}
    \FunctionTok{layer\_dropout}\NormalTok{(}\FloatTok{0.1}\NormalTok{) }\SpecialCharTok{\%\textgreater{}\%}
    \CommentTok{\# layer\_dense(units = 16, activation = \textquotesingle{}relu\textquotesingle{}) \%\textgreater{}\%}
\FunctionTok{layer\_dense}\NormalTok{(}\AttributeTok{units =} \DecValTok{1}\NormalTok{, }\AttributeTok{activation =} \StringTok{"sigmoid"}\NormalTok{, }\AttributeTok{name =} \StringTok{"classifier"}\NormalTok{)}

\CommentTok{\# summary(model)}
\end{Highlighting}
\end{Shaded}

\begin{Shaded}
\begin{Highlighting}[]
\NormalTok{model }\OtherTok{\textless{}{-}} \FunctionTok{keras\_model}\NormalTok{(input, output)}
\end{Highlighting}
\end{Shaded}

Loss function

Since this is a binary classification problem and the model outputs a
probability (a single-unit layer), you'll use ``binary\_crossentropy''
loss function.

Optimizer

For fine-tuning, let's use the same optimizer that BERT was originally
trained with: the ``Adaptive Moments'' (Adam). This optimizer minimizes
the prediction loss and does regularization by weight decay (not using
moments), which is also known as
\href{https://arxiv.org/abs/1711.05101}{AdamW}.

We will use the AdamW optimizer from
\href{https://github.com/tensorflow/models}{tensorflow/models}.

For the learning rate (init\_lr), you will use the same schedule as BERT
pre-training: linear decay of a notional initial learning rate, prefixed
with a linear warm-up phase over the first 10\% of training steps
(num\_warmup\_steps). In line with the BERT paper, the initial learning
rate is smaller for fine-tuning (best of 5e-5, 3e-5, 2e-5).

\begin{Shaded}
\begin{Highlighting}[]
\NormalTok{epochs }\OtherTok{=} \DecValTok{5}
\NormalTok{steps\_per\_epoch }\OtherTok{\textless{}{-}} \FloatTok{2e+06}
\NormalTok{num\_train\_steps }\OtherTok{\textless{}{-}}\NormalTok{ steps\_per\_epoch }\SpecialCharTok{*}\NormalTok{ epochs}
\NormalTok{num\_warmup\_steps }\OtherTok{\textless{}{-}} \FunctionTok{as.integer}\NormalTok{(}\FloatTok{0.1} \SpecialCharTok{*}\NormalTok{ num\_train\_steps)}

\NormalTok{init\_lr }\OtherTok{\textless{}{-}} \FloatTok{3e{-}05}
\NormalTok{opt }\OtherTok{\textless{}{-}}\NormalTok{ o\_nlp}\SpecialCharTok{$}\NormalTok{optimization}\SpecialCharTok{$}\FunctionTok{create\_optimizer}\NormalTok{(}\AttributeTok{init\_lr =}\NormalTok{ init\_lr, }\AttributeTok{num\_train\_steps =}\NormalTok{ num\_train\_steps,}
    \AttributeTok{num\_warmup\_steps =}\NormalTok{ num\_warmup\_steps, }\AttributeTok{optimizer\_type =} \StringTok{"adamw"}\NormalTok{)}

\NormalTok{model }\SpecialCharTok{\%\textgreater{}\%}
    \FunctionTok{compile}\NormalTok{(}\AttributeTok{loss =} \StringTok{"binary\_crossentropy"}\NormalTok{, }\AttributeTok{optimizer =}\NormalTok{ opt, }\AttributeTok{metrics =} \StringTok{"accuracy"}\NormalTok{)}

\FunctionTok{summary}\NormalTok{(model)}
\NormalTok{Model}\SpecialCharTok{:} \StringTok{"model"}
\NormalTok{\_\_\_\_\_\_\_\_\_\_\_\_\_\_\_\_\_\_\_\_\_\_\_\_\_\_\_\_\_\_\_\_\_\_\_\_\_\_\_\_\_\_\_\_\_\_\_\_\_\_\_\_\_\_\_\_\_\_\_\_\_\_\_\_\_\_\_\_\_\_\_\_\_\_\_\_\_\_\_\_}
\FunctionTok{Layer}\NormalTok{ (type)              Output Shape      Param }\CommentTok{\#  Connected to               }
\SpecialCharTok{==}\ErrorTok{==============================================================================}
\FunctionTok{text}\NormalTok{ (InputLayer)         [(None,)]         }\DecValTok{0}                                   
\NormalTok{\_\_\_\_\_\_\_\_\_\_\_\_\_\_\_\_\_\_\_\_\_\_\_\_\_\_\_\_\_\_\_\_\_\_\_\_\_\_\_\_\_\_\_\_\_\_\_\_\_\_\_\_\_\_\_\_\_\_\_\_\_\_\_\_\_\_\_\_\_\_\_\_\_\_\_\_\_\_\_\_}
\FunctionTok{preprocessing}\NormalTok{ (KerasLayer \{}\StringTok{\textquotesingle{}input\_mask\textquotesingle{}}\SpecialCharTok{:}\NormalTok{ (N }\DecValTok{0}\NormalTok{        text[}\DecValTok{0}\NormalTok{][}\DecValTok{0}\NormalTok{]                 }
\NormalTok{\_\_\_\_\_\_\_\_\_\_\_\_\_\_\_\_\_\_\_\_\_\_\_\_\_\_\_\_\_\_\_\_\_\_\_\_\_\_\_\_\_\_\_\_\_\_\_\_\_\_\_\_\_\_\_\_\_\_\_\_\_\_\_\_\_\_\_\_\_\_\_\_\_\_\_\_\_\_\_\_}
\FunctionTok{BERT\_encoder}\NormalTok{ (KerasLayer) \{}\StringTok{\textquotesingle{}default\textquotesingle{}}\SpecialCharTok{:}\NormalTok{ (None }\DecValTok{28763649}\NormalTok{ preprocessing[}\DecValTok{0}\NormalTok{][}\DecValTok{0}\NormalTok{]        }
\NormalTok{                                                     preprocessing[}\DecValTok{0}\NormalTok{][}\DecValTok{1}\NormalTok{]        }
\NormalTok{                                                     preprocessing[}\DecValTok{0}\NormalTok{][}\DecValTok{2}\NormalTok{]        }
\NormalTok{\_\_\_\_\_\_\_\_\_\_\_\_\_\_\_\_\_\_\_\_\_\_\_\_\_\_\_\_\_\_\_\_\_\_\_\_\_\_\_\_\_\_\_\_\_\_\_\_\_\_\_\_\_\_\_\_\_\_\_\_\_\_\_\_\_\_\_\_\_\_\_\_\_\_\_\_\_\_\_\_}
\FunctionTok{dropout\_4}\NormalTok{ (Dropout)       (None, }\DecValTok{512}\NormalTok{)       }\DecValTok{0}\NormalTok{        BERT\_encoder[}\DecValTok{0}\NormalTok{][}\DecValTok{5}\NormalTok{]         }
\NormalTok{\_\_\_\_\_\_\_\_\_\_\_\_\_\_\_\_\_\_\_\_\_\_\_\_\_\_\_\_\_\_\_\_\_\_\_\_\_\_\_\_\_\_\_\_\_\_\_\_\_\_\_\_\_\_\_\_\_\_\_\_\_\_\_\_\_\_\_\_\_\_\_\_\_\_\_\_\_\_\_\_}
\FunctionTok{classifier}\NormalTok{ (Dense)        (None, }\DecValTok{1}\NormalTok{)         }\DecValTok{513}\NormalTok{      dropout\_4[}\DecValTok{0}\NormalTok{][}\DecValTok{0}\NormalTok{]            }
\SpecialCharTok{==}\ErrorTok{==============================================================================}
\NormalTok{Total params}\SpecialCharTok{:} \DecValTok{28}\NormalTok{,}\DecValTok{764}\NormalTok{,}\DecValTok{162}
\NormalTok{Trainable params}\SpecialCharTok{:} \DecValTok{28}\NormalTok{,}\DecValTok{764}\NormalTok{,}\DecValTok{161}
\NormalTok{Non}\SpecialCharTok{{-}}\NormalTok{trainable params}\SpecialCharTok{:} \DecValTok{1}
\NormalTok{\_\_\_\_\_\_\_\_\_\_\_\_\_\_\_\_\_\_\_\_\_\_\_\_\_\_\_\_\_\_\_\_\_\_\_\_\_\_\_\_\_\_\_\_\_\_\_\_\_\_\_\_\_\_\_\_\_\_\_\_\_\_\_\_\_\_\_\_\_\_\_\_\_\_\_\_\_\_\_\_}
\end{Highlighting}
\end{Shaded}

REMEMBER to change tr\_count back to 10000 for better training.

Try a sample/subset to train/test code, and to reduce training time due
to resource constraints use a smaller tr\_count below.

\begin{Shaded}
\begin{Highlighting}[]
\CommentTok{\# 10000 will take approx 40 mins per epoch on my gpu/mem etc 1000 will take}
\CommentTok{\# approx 4 mins per epoch on my gpu/mem etc}

\NormalTok{tr\_count }\OtherTok{\textless{}{-}} \DecValTok{10000}
\NormalTok{take\_tr }\OtherTok{\textless{}{-}} \FloatTok{0.8} \SpecialCharTok{*}\NormalTok{ tr\_count}
\NormalTok{train\_slice }\OtherTok{\textless{}{-}}\NormalTok{ train\_dataset }\SpecialCharTok{\%\textgreater{}\%}
    \FunctionTok{dataset\_shuffle\_and\_repeat}\NormalTok{(}\AttributeTok{buffer\_size =}\NormalTok{ take\_tr }\SpecialCharTok{*}\NormalTok{ batch\_size) }\SpecialCharTok{\%\textgreater{}\%}
    \FunctionTok{dataset\_take}\NormalTok{(take\_tr)}

\NormalTok{take\_val }\OtherTok{\textless{}{-}} \FloatTok{0.2} \SpecialCharTok{*}\NormalTok{ tr\_count}
\NormalTok{val\_slice }\OtherTok{\textless{}{-}}\NormalTok{ val\_dataset }\SpecialCharTok{\%\textgreater{}\%}
    \FunctionTok{dataset\_shuffle\_and\_repeat}\NormalTok{(}\AttributeTok{buffer\_size =}\NormalTok{ take\_val }\SpecialCharTok{*}\NormalTok{ batch\_size) }\SpecialCharTok{\%\textgreater{}\%}
    \FunctionTok{dataset\_take}\NormalTok{(take\_val)}
\end{Highlighting}
\end{Shaded}

\begin{Shaded}
\begin{Highlighting}[]
\NormalTok{epochs }\OtherTok{\textless{}{-}} \DecValTok{5}
\NormalTok{seed }\OtherTok{=} \DecValTok{42}

\NormalTok{history }\OtherTok{\textless{}{-}}\NormalTok{ model }\SpecialCharTok{\%\textgreater{}\%}
    \FunctionTok{fit}\NormalTok{(train\_slice, }\AttributeTok{epochs =}\NormalTok{ epochs, }\AttributeTok{validation\_data =}\NormalTok{ val\_slice, }\AttributeTok{initial\_epoch =} \DecValTok{0}\NormalTok{,}
        \AttributeTok{verbose =} \DecValTok{2}\NormalTok{)}
\end{Highlighting}
\end{Shaded}

``BERT Model Fit History using validation\_data slice'' Figure
\ref{fig:model_20}

\begin{figure}
\centering
\includegraphics{figures/keras_model_fit_history-1.pdf}
\caption{BERT Model Fit History using validation\_data
slice\label{fig:model_20}}
\end{figure}

Evaluate the model

Let's see how the model performs. Two values will be returned. Loss (a
number which represents the error, lower values are better), and
accuracy.

Takes too long, so skipping it for now. Using test\_slice instead.

\begin{Shaded}
\begin{Highlighting}[]
\NormalTok{model }\SpecialCharTok{\%\textgreater{}\%}
    \FunctionTok{evaluate}\NormalTok{(test\_dataset)}
\end{Highlighting}
\end{Shaded}

Using test\_slice instead.

\begin{Shaded}
\begin{Highlighting}[]
\NormalTok{test\_slice }\OtherTok{\textless{}{-}}\NormalTok{ test\_dataset }\SpecialCharTok{\%\textgreater{}\%}
    \FunctionTok{dataset\_take}\NormalTok{(}\DecValTok{100}\NormalTok{)}

\NormalTok{model }\SpecialCharTok{\%\textgreater{}\%}
    \FunctionTok{evaluate}\NormalTok{(test\_slice)}
\NormalTok{     loss  accuracy }
\FloatTok{0.2556886} \FloatTok{0.9012500} 
\end{Highlighting}
\end{Shaded}

\newpage

\hypertarget{results-4}{%
\subsection{Results}\label{results-4}}

BERT model results Table \ref{tbl:bert_results_table}:

\begin{longtable}[]{@{}llrr@{}}
\caption{BERT Model
Results\label{tbl:bert_results_table}}\tabularnewline
\toprule
Index & Method & Accuracy & Loss \\
\midrule
\endfirsthead
\toprule
Index & Method & Accuracy & Loss \\
\midrule
\endhead
1 & BLM & 0.8897064 & NA \\
2 & DNN & 0.8960983 & 0.2790776 \\
3 & CNN & 0.8982586 & 0.2526298 \\
4 & sepCNN & 0.9051675 & 0.2408866 \\
5 & BERT & 0.9051719 & 0.2386644 \\
\bottomrule
\end{longtable}

\begin{Shaded}
\begin{Highlighting}[]
\CommentTok{\# Stop the clock}
\CommentTok{\# proc.time() {-} ptm}
\FunctionTok{Sys.time}\NormalTok{()}
\end{Highlighting}
\end{Shaded}

\begin{verbatim}
## [1] "2021-07-28 05:13:36 EDT"
\end{verbatim}

\begin{center}\rule{0.5\linewidth}{0.5pt}\end{center}

\newpage

\hypertarget{conclusion}{%
\section{Conclusion}\label{conclusion}}

In this project, we attempted to significantly simplify the process of
selecting a text classification model. For a given dataset, our goal was
to find the algorithm that achieved close to maximum accuracy while
minimizing computation time required for training.

CNNs are a type of neural network that can learn local spatial patterns.
They essentially perform feature extraction, which can then be used
efficiently in later layers of a network. Their simplicity and fast
running time, compared to other models, makes them excellent candidates
for supervised models for text.

Based on our results and inspite of using only a fraction of our data
due to (my) resource limitations, we agree with Google and conclude that
sepCNN's and/or BERT helped us achieve our goal of simplicity, minimum
compute time and maximum accuracy.

As of now, my future attempts in ML will be in NLP related activities.

\begin{center}\rule{0.5\linewidth}{0.5pt}\end{center}

\newpage

\hypertarget{appendix-all-code-for-this-report}{%
\section{Appendix: All code for this
report}\label{appendix-all-code-for-this-report}}

\begin{Shaded}
\begin{Highlighting}[]
\NormalTok{knitr}\SpecialCharTok{::}\NormalTok{knit\_hooks}\SpecialCharTok{$}\FunctionTok{set}\NormalTok{(}\AttributeTok{time\_it =} \FunctionTok{local}\NormalTok{(\{}
\NormalTok{  now }\OtherTok{\textless{}{-}} \ConstantTok{NULL}
  \ControlFlowTok{function}\NormalTok{(before, options) \{}
    \ControlFlowTok{if}\NormalTok{ (before) \{}
      \CommentTok{\# record the current time before each chunk}
\NormalTok{      now }\OtherTok{\textless{}\textless{}{-}} \FunctionTok{Sys.time}\NormalTok{()}
\NormalTok{    \} }\ControlFlowTok{else}\NormalTok{ \{}
      \CommentTok{\# calculate the time difference after a chunk}
\NormalTok{      res }\OtherTok{\textless{}{-}} \FunctionTok{difftime}\NormalTok{(}\FunctionTok{Sys.time}\NormalTok{(), now)}
      \CommentTok{\# return a character string to show the time}
      \CommentTok{\# paste("Time for this code chunk to run:", res)}
      \FunctionTok{paste}\NormalTok{(}\StringTok{"Time for the chunk"}\NormalTok{, options}\SpecialCharTok{$}\NormalTok{label, }\StringTok{"to run:"}\NormalTok{, res)}
\NormalTok{    \}}
\NormalTok{  \}}
\NormalTok{\}))}

\CommentTok{\# knit\_hooks$get("inline")}
\CommentTok{\# knitr::opts\_chunk$set(fig.pos = "!H", out.extra = "")}
\NormalTok{knitr}\SpecialCharTok{::}\NormalTok{opts\_chunk}\SpecialCharTok{$}\FunctionTok{set}\NormalTok{(}\AttributeTok{echo =} \ConstantTok{TRUE}\NormalTok{,}
                      \AttributeTok{fig.path =} \StringTok{"figures/"}\NormalTok{)}

\CommentTok{\# Beware, using the "time\_it" hook messes up fig.cap, \textbackslash{}label, \textbackslash{}ref}
\CommentTok{\# knitr::opts\_chunk$set(time\_it = TRUE)}
\CommentTok{\#knitr::opts\_chunk$set(eval = FALSE)}
\FunctionTok{options}\NormalTok{(}\AttributeTok{tinytex.verbose =} \ConstantTok{TRUE}\NormalTok{)}

\CommentTok{\# set pandoc stack size}
\NormalTok{stack\_size }\OtherTok{\textless{}{-}} \FunctionTok{getOption}\NormalTok{(}\StringTok{"pandoc.stack.size"}\NormalTok{, }\AttributeTok{default =} \StringTok{"100000000"}\NormalTok{)}
\NormalTok{args }\OtherTok{\textless{}{-}} \FunctionTok{c}\NormalTok{(}\FunctionTok{c}\NormalTok{(}\StringTok{"+RTS"}\NormalTok{, }\FunctionTok{paste0}\NormalTok{(}\StringTok{"{-}K"}\NormalTok{, stack\_size), }\StringTok{"{-}RTS"}\NormalTok{), args)}
\CommentTok{\# library(dplyr)}
\CommentTok{\# library(tidyr)}
\CommentTok{\# library(purrr)}
\CommentTok{\# library(readr)}
\FunctionTok{library}\NormalTok{(tidyverse)}
\FunctionTok{library}\NormalTok{(textrecipes)}
\FunctionTok{library}\NormalTok{(tidymodels)}
\FunctionTok{library}\NormalTok{(tidytext)}
\FunctionTok{library}\NormalTok{(ngram)}
\FunctionTok{library}\NormalTok{(keras)}
\FunctionTok{library}\NormalTok{(stopwords)}

\CommentTok{\# Used in Baseline model}
\FunctionTok{library}\NormalTok{(hardhat)}

\CommentTok{\# BERT setup in its own section}
\CommentTok{\# library(keras)}
\CommentTok{\# library(tfdatasets)}
\CommentTok{\# library(reticulate)}
\CommentTok{\# library(tidyverse)}
\CommentTok{\# library(lubridate)}
\CommentTok{\# library(tfhub)}
\CommentTok{\# import("tensorflow\_text")}
\CommentTok{\# o\_nlp \textless{}{-} import("official.nlp")}
\CommentTok{\# }
\CommentTok{\# Sys.setenv(TFHUB\_CACHE\_DIR="C:/Users/bijoor/.cache/tfhub\_modules")}
\CommentTok{\# Sys.getenv("TFHUB\_CACHE\_DIR")}

\FunctionTok{set.seed}\NormalTok{(}\DecValTok{234}\NormalTok{)}

\CommentTok{\# Start the clock!}
\CommentTok{\# ptm \textless{}{-} proc.time()}
\FunctionTok{Sys.time}\NormalTok{()}
  \FunctionTok{library}\NormalTok{(ggplot2)}
  \FunctionTok{library}\NormalTok{(kableExtra)}
\FunctionTok{untar}\NormalTok{(}\StringTok{"amazon\_review\_polarity\_csv.tar.gz"}\NormalTok{,}\AttributeTok{list=}\ConstantTok{TRUE}\NormalTok{)  }\DocumentationTok{\#\# check contents}
\FunctionTok{untar}\NormalTok{(}\StringTok{"amazon\_review\_polarity\_csv.tar.gz"}\NormalTok{)}
\NormalTok{train\_file\_path }\OtherTok{\textless{}{-}} \FunctionTok{file.path}\NormalTok{(}\StringTok{"amazon\_review\_polarity\_csv/train.csv"}\NormalTok{)}

\NormalTok{test\_file\_path }\OtherTok{\textless{}{-}} \FunctionTok{file.path}\NormalTok{(}\StringTok{"amazon\_review\_polarity\_csv/test.csv"}\NormalTok{)}

\CommentTok{\# read data, ensure "utf{-}8" encoding, add column names, exclude rows with missing values(NA)}
\NormalTok{amazon\_orig\_train }\OtherTok{\textless{}{-}}\NormalTok{ readr}\SpecialCharTok{::}\FunctionTok{read\_csv}\NormalTok{(}
\NormalTok{  train\_file\_path,}
  \CommentTok{\# skip = 0,}
  \AttributeTok{col\_names =} \FunctionTok{c}\NormalTok{(}\StringTok{"label"}\NormalTok{, }\StringTok{"title"}\NormalTok{, }\StringTok{"text"}\NormalTok{),}
  \AttributeTok{locale =} \FunctionTok{locale}\NormalTok{(}\AttributeTok{encoding =} \StringTok{"UTF{-}8"}\NormalTok{)) }\SpecialCharTok{\%\textgreater{}\%} \FunctionTok{na.omit}\NormalTok{()}

\CommentTok{\# change labels from (1,2) to (0,1) {-} easier for binary classification}
\NormalTok{amazon\_orig\_train}\SpecialCharTok{$}\NormalTok{label[amazon\_orig\_train}\SpecialCharTok{$}\NormalTok{label}\SpecialCharTok{==}\DecValTok{1}\NormalTok{] }\OtherTok{\textless{}{-}} \DecValTok{0}
\NormalTok{amazon\_orig\_train}\SpecialCharTok{$}\NormalTok{label[amazon\_orig\_train}\SpecialCharTok{$}\NormalTok{label}\SpecialCharTok{==}\DecValTok{2}\NormalTok{] }\OtherTok{\textless{}{-}} \DecValTok{1}

\CommentTok{\# removed numbers as they were too many and did not contribute any info}

\CommentTok{\# amazon\_orig\_train$text \textless{}{-} str\_replace\_all(amazon\_orig\_train$text,"[\^{}([[:alnum:]\_])]"," ") \%\textgreater{}\% trimws() \%\textgreater{}\% str\_squish()}
\CommentTok{\# }
\CommentTok{\# amazon\_orig\_train$title \textless{}{-} str\_replace\_all(amazon\_orig\_train$title,"[\^{}([[:alnum:]\_])]"," ") \%\textgreater{}\% trimws() \%\textgreater{}\% str\_squish()}

\CommentTok{\# remove leading/trailing whitespace (trimws)}
\CommentTok{\# trim whitespace from a string (str\_squish)}
\CommentTok{\# replace non alphabet chars with space}

\NormalTok{amazon\_orig\_train}\SpecialCharTok{$}\NormalTok{text }\OtherTok{\textless{}{-}} \FunctionTok{str\_replace\_all}\NormalTok{(amazon\_orig\_train}\SpecialCharTok{$}\NormalTok{text,}\StringTok{"[\^{}([[:alpha:]\_])]"}\NormalTok{,}\StringTok{" "}\NormalTok{) }\SpecialCharTok{\%\textgreater{}\%} \FunctionTok{trimws}\NormalTok{() }\SpecialCharTok{\%\textgreater{}\%} \FunctionTok{str\_squish}\NormalTok{()}

\NormalTok{amazon\_orig\_train}\SpecialCharTok{$}\NormalTok{title }\OtherTok{\textless{}{-}} \FunctionTok{str\_replace\_all}\NormalTok{(amazon\_orig\_train}\SpecialCharTok{$}\NormalTok{title,}\StringTok{"[\^{}([[:alpha:]\_])]"}\NormalTok{,}\StringTok{" "}\NormalTok{) }\SpecialCharTok{\%\textgreater{}\%} \FunctionTok{trimws}\NormalTok{() }\SpecialCharTok{\%\textgreater{}\%} \FunctionTok{str\_squish}\NormalTok{()}

\CommentTok{\# create a validation set for training purposes}
\NormalTok{ids\_train }\OtherTok{\textless{}{-}} \FunctionTok{sample.int}\NormalTok{(}\FunctionTok{nrow}\NormalTok{(amazon\_orig\_train), }\AttributeTok{size =} \FloatTok{0.8}\SpecialCharTok{*}\FunctionTok{nrow}\NormalTok{(amazon\_orig\_train))}
\NormalTok{amazon\_train }\OtherTok{\textless{}{-}}\NormalTok{ amazon\_orig\_train[ids\_train,]}
\NormalTok{amazon\_val }\OtherTok{\textless{}{-}}\NormalTok{ amazon\_orig\_train[}\SpecialCharTok{{-}}\NormalTok{ids\_train,]}

\FunctionTok{head}\NormalTok{(amazon\_train)}

\CommentTok{\# save cleaned up data for later use}
\FunctionTok{write\_csv}\NormalTok{(amazon\_train,}\StringTok{"amazon\_review\_polarity\_csv/amazon\_train.csv"}\NormalTok{, }\AttributeTok{col\_names =} \ConstantTok{TRUE}\NormalTok{)}
\FunctionTok{write\_csv}\NormalTok{(amazon\_val,}\StringTok{"amazon\_review\_polarity\_csv/amazon\_val.csv"}\NormalTok{, }\AttributeTok{col\_names =} \ConstantTok{TRUE}\NormalTok{)}

\CommentTok{\# {-}{-}{-}{-}{-}{-}{-}{-}{-}{-}{-}{-}{-}{-}{-}{-}{-}{-}{-}{-}{-}{-}{-}{-}{-}{-}{-}{-}{-}{-}{-}{-}{-}{-}{-}{-}{-}{-}{-}{-}{-}{-}{-}{-}{-}{-}{-}}
\CommentTok{\# read data, ensure "utf{-}8" encoding, add column names, exclude rows with missing values(NA)}
\NormalTok{amazon\_orig\_test }\OtherTok{\textless{}{-}}\NormalTok{ readr}\SpecialCharTok{::}\FunctionTok{read\_csv}\NormalTok{(}
\NormalTok{  test\_file\_path,}
  \CommentTok{\# skip = 0,}
  \AttributeTok{col\_names =} \FunctionTok{c}\NormalTok{(}\StringTok{"label"}\NormalTok{, }\StringTok{"title"}\NormalTok{, }\StringTok{"text"}\NormalTok{),}
  \AttributeTok{locale =} \FunctionTok{locale}\NormalTok{(}\AttributeTok{encoding =} \StringTok{"UTF{-}8"}\NormalTok{)) }\SpecialCharTok{\%\textgreater{}\%} \FunctionTok{na.omit}\NormalTok{()}

\CommentTok{\# change labels from (1,2) to (0,1) {-} easier for binary classification}
\NormalTok{amazon\_orig\_test}\SpecialCharTok{$}\NormalTok{label[amazon\_orig\_test}\SpecialCharTok{$}\NormalTok{label}\SpecialCharTok{==}\DecValTok{1}\NormalTok{] }\OtherTok{\textless{}{-}} \DecValTok{0}
\NormalTok{amazon\_orig\_test}\SpecialCharTok{$}\NormalTok{label[amazon\_orig\_test}\SpecialCharTok{$}\NormalTok{label}\SpecialCharTok{==}\DecValTok{2}\NormalTok{] }\OtherTok{\textless{}{-}} \DecValTok{1}

\CommentTok{\# remove leading/trailing whitespace (trimws)}
\CommentTok{\# trim whitespace from a string (str\_squish)}
\CommentTok{\# replace non alphabet chars with space}

\NormalTok{amazon\_orig\_test}\SpecialCharTok{$}\NormalTok{text }\OtherTok{\textless{}{-}} \FunctionTok{str\_replace\_all}\NormalTok{(amazon\_orig\_test}\SpecialCharTok{$}\NormalTok{text,}\StringTok{"[\^{}([[:alpha:]\_])]"}\NormalTok{,}\StringTok{" "}\NormalTok{) }\SpecialCharTok{\%\textgreater{}\%} \FunctionTok{trimws}\NormalTok{() }\SpecialCharTok{\%\textgreater{}\%} \FunctionTok{str\_squish}\NormalTok{()}

\NormalTok{amazon\_orig\_test}\SpecialCharTok{$}\NormalTok{title }\OtherTok{\textless{}{-}} \FunctionTok{str\_replace\_all}\NormalTok{(amazon\_orig\_test}\SpecialCharTok{$}\NormalTok{title,}\StringTok{"[\^{}([[:alpha:]\_])]"}\NormalTok{,}\StringTok{" "}\NormalTok{) }\SpecialCharTok{\%\textgreater{}\%} \FunctionTok{trimws}\NormalTok{() }\SpecialCharTok{\%\textgreater{}\%} \FunctionTok{str\_squish}\NormalTok{()}

\CommentTok{\# amazon\_orig\_test$text \textless{}{-} str\_replace\_all(amazon\_orig\_test$text,"[\^{}([[:alnum:]\_])]"," ") \%\textgreater{}\% trimws() \%\textgreater{}\% str\_squish()}
\CommentTok{\# }
\CommentTok{\# amazon\_orig\_test$title \textless{}{-} str\_replace\_all(amazon\_orig\_test$title,"[\^{}([[:alnum:]\_])]"," ") \%\textgreater{}\% trimws() \%\textgreater{}\% str\_squish()}

\FunctionTok{head}\NormalTok{(amazon\_orig\_test)}

\CommentTok{\# save cleaned up data for later use}
\FunctionTok{write\_csv}\NormalTok{(amazon\_orig\_test,}\StringTok{"amazon\_review\_polarity\_csv/amazon\_test.csv"}\NormalTok{, }\AttributeTok{col\_names =} \ConstantTok{TRUE}\NormalTok{)}

\FunctionTok{rm}\NormalTok{(amazon\_orig\_train, amazon\_orig\_test)}
\FunctionTok{rm}\NormalTok{(ids\_train, test\_file\_path, train\_file\_path)}

\CommentTok{\# free unused R memory}
\FunctionTok{gc}\NormalTok{()}
\DocumentationTok{\#\#\#\# To be deleted later}
\NormalTok{amazon\_train }\OtherTok{\textless{}{-}}\NormalTok{ readr}\SpecialCharTok{::}\FunctionTok{read\_csv}\NormalTok{(}\StringTok{"amazon\_review\_polarity\_csv/amazon\_train.csv"}\NormalTok{)}
\FunctionTok{glimpse}\NormalTok{(amazon\_train)}
\CommentTok{\# head(amazon\_train)}
\FunctionTok{kable}\NormalTok{(amazon\_train[}\DecValTok{1}\SpecialCharTok{:}\DecValTok{10}\NormalTok{,], }\StringTok{"latex"}\NormalTok{, }\AttributeTok{escape=}\ConstantTok{FALSE}\NormalTok{, }\AttributeTok{booktabs=}\ConstantTok{TRUE}\NormalTok{, }\AttributeTok{linesep=}\StringTok{""}\NormalTok{, }\AttributeTok{caption=}\StringTok{"Amazon Train  data}\SpecialCharTok{\textbackslash{}\textbackslash{}}\StringTok{label\{tbl:amazon\_train\}"}\NormalTok{) }\SpecialCharTok{\%\textgreater{}\%}
    \FunctionTok{kable\_styling}\NormalTok{(}\AttributeTok{latex\_options=}\FunctionTok{c}\NormalTok{(}\StringTok{"HOLD\_position"}\NormalTok{), }\AttributeTok{font\_size=}\DecValTok{6}\NormalTok{)}
  \CommentTok{\# kable\_styling(full\_width = F)}
\FunctionTok{unique}\NormalTok{(amazon\_train}\SpecialCharTok{$}\NormalTok{label)}
\NormalTok{(num\_samples }\OtherTok{\textless{}{-}} \FunctionTok{nrow}\NormalTok{(amazon\_train))}
\NormalTok{(num\_classes }\OtherTok{\textless{}{-}} \FunctionTok{length}\NormalTok{(}\FunctionTok{unique}\NormalTok{(amazon\_train}\SpecialCharTok{$}\NormalTok{label)))}
\CommentTok{\# Pretty Balanced classes}
\NormalTok{(num\_samples\_per\_class }\OtherTok{\textless{}{-}}\NormalTok{ amazon\_train }\SpecialCharTok{\%\textgreater{}\%} \FunctionTok{count}\NormalTok{(label))}
\CommentTok{\# break up the strings in each row by " "}
\NormalTok{temp }\OtherTok{\textless{}{-}} \FunctionTok{strsplit}\NormalTok{(amazon\_train}\SpecialCharTok{$}\NormalTok{text, }\AttributeTok{split=}\StringTok{" "}\NormalTok{)}

\CommentTok{\# sapply(temp[c(1:3)], length)}
\CommentTok{\# count the number of words as the length of the vectors}
\NormalTok{amazon\_train\_text\_wordCount }\OtherTok{\textless{}{-}} \FunctionTok{sapply}\NormalTok{(temp, length)}

\NormalTok{(mean\_num\_words\_per\_sample }\OtherTok{\textless{}{-}} \FunctionTok{mean}\NormalTok{(amazon\_train\_text\_wordCount))}

\NormalTok{(median\_num\_words\_per\_sample }\OtherTok{\textless{}{-}} \FunctionTok{median}\NormalTok{(amazon\_train\_text\_wordCount))}
\CommentTok{\# Frequency distribution of words(ngrams)}
\NormalTok{train\_words }\OtherTok{\textless{}{-}}\NormalTok{ amazon\_train }\SpecialCharTok{\%\textgreater{}\%} \FunctionTok{unnest\_tokens}\NormalTok{(word, text) }\SpecialCharTok{\%\textgreater{}\%} \FunctionTok{count}\NormalTok{(word,}\AttributeTok{sort =} \ConstantTok{TRUE}\NormalTok{)}

\NormalTok{total\_words }\OtherTok{\textless{}{-}}\NormalTok{ train\_words }\SpecialCharTok{\%\textgreater{}\%}
    \FunctionTok{summarize}\NormalTok{(}\AttributeTok{total =} \FunctionTok{sum}\NormalTok{(n))}

\CommentTok{\# Zipf’s law states that the frequency that a word appears is inversely proportional to its rank.}
\NormalTok{train\_words }\OtherTok{\textless{}{-}}\NormalTok{ train\_words }\SpecialCharTok{\%\textgreater{}\%}
    \FunctionTok{mutate}\NormalTok{(total\_words) }\SpecialCharTok{\%\textgreater{}\%}
     \FunctionTok{mutate}\NormalTok{(}\AttributeTok{rank =} \FunctionTok{row\_number}\NormalTok{(),}
         \StringTok{\textasciigrave{}}\AttributeTok{term frequency}\StringTok{\textasciigrave{}} \OtherTok{=}\NormalTok{ n}\SpecialCharTok{/}\NormalTok{total)}
\CommentTok{\# head(train\_words)}
\FunctionTok{kable}\NormalTok{(train\_words[}\DecValTok{1}\SpecialCharTok{:}\DecValTok{10}\NormalTok{,], }\StringTok{"latex"}\NormalTok{, }\AttributeTok{escape=}\ConstantTok{FALSE}\NormalTok{, }\AttributeTok{booktabs=}\ConstantTok{TRUE}\NormalTok{, }\AttributeTok{linesep=}\StringTok{""}\NormalTok{, }\AttributeTok{caption=}\StringTok{"Frequency distribution of words}\SpecialCharTok{\textbackslash{}\textbackslash{}}\StringTok{label\{tbl:train\_words\}"}\NormalTok{) }\CommentTok{\#\%\textgreater{}\%}
    \CommentTok{\# kable\_styling(latex\_options=c("HOLD\_position"), font\_size=6)}
\NormalTok{train\_words }\SpecialCharTok{\%\textgreater{}\%}
  \FunctionTok{top\_n}\NormalTok{(}\DecValTok{25}\NormalTok{, n) }\SpecialCharTok{\%\textgreater{}\%}
  \FunctionTok{ggplot}\NormalTok{(}\FunctionTok{aes}\NormalTok{(}\FunctionTok{reorder}\NormalTok{(word,n),n)) }\SpecialCharTok{+}
  \FunctionTok{geom\_col}\NormalTok{(}\AttributeTok{binwidth =} \DecValTok{1}\NormalTok{, }\AttributeTok{alpha =} \FloatTok{0.8}\NormalTok{) }\SpecialCharTok{+}
   \FunctionTok{coord\_flip}\NormalTok{() }\SpecialCharTok{+}
  \FunctionTok{labs}\NormalTok{(}\AttributeTok{y=}\StringTok{"n {-} Frequency distribution of words(ngrams)"}\NormalTok{,}
       \AttributeTok{x=}\StringTok{"Top 25 words"}\NormalTok{)}
\FunctionTok{length}\NormalTok{(}\FunctionTok{stopwords}\NormalTok{(}\AttributeTok{source =} \StringTok{"smart"}\NormalTok{))}
\FunctionTok{length}\NormalTok{(}\FunctionTok{stopwords}\NormalTok{(}\AttributeTok{source =} \StringTok{"snowball"}\NormalTok{))}
\FunctionTok{length}\NormalTok{(}\FunctionTok{stopwords}\NormalTok{(}\AttributeTok{source =} \StringTok{"stopwords{-}iso"}\NormalTok{))}
\NormalTok{mystopwords }\OtherTok{\textless{}{-}} \FunctionTok{c}\NormalTok{(}\StringTok{"s"}\NormalTok{, }\StringTok{"t"}\NormalTok{, }\StringTok{"m"}\NormalTok{, }\StringTok{"ve"}\NormalTok{, }\StringTok{"re"}\NormalTok{, }\StringTok{"d"}\NormalTok{, }\StringTok{"ll"}\NormalTok{)}

\CommentTok{\# Frequency distribution of words(ngrams)}
\NormalTok{train\_words\_sw }\OtherTok{\textless{}{-}}\NormalTok{ amazon\_train }\SpecialCharTok{\%\textgreater{}\%} \FunctionTok{unnest\_tokens}\NormalTok{(word, text) }\SpecialCharTok{\%\textgreater{}\%}
    \FunctionTok{anti\_join}\NormalTok{(}\FunctionTok{get\_stopwords}\NormalTok{(}\AttributeTok{source =} \StringTok{"stopwords{-}iso"}\NormalTok{))}\SpecialCharTok{\%\textgreater{}\%}
    \FunctionTok{filter}\NormalTok{(}\SpecialCharTok{!}\NormalTok{(word }\SpecialCharTok{\%in\%}\NormalTok{ mystopwords)) }\SpecialCharTok{\%\textgreater{}\%}
    \FunctionTok{count}\NormalTok{(word,}\AttributeTok{sort =} \ConstantTok{TRUE}\NormalTok{)}

\NormalTok{total\_words\_sw }\OtherTok{\textless{}{-}}\NormalTok{ train\_words\_sw }\SpecialCharTok{\%\textgreater{}\%}
    \FunctionTok{summarize}\NormalTok{(}\AttributeTok{total =} \FunctionTok{sum}\NormalTok{(n))}

\CommentTok{\# Zipf’s law states that the frequency that a word appears is inversely proportional to its rank.}

\NormalTok{train\_words\_sw }\OtherTok{\textless{}{-}}\NormalTok{ train\_words\_sw }\SpecialCharTok{\%\textgreater{}\%}
    \FunctionTok{mutate}\NormalTok{(total\_words\_sw) }\SpecialCharTok{\%\textgreater{}\%}
     \FunctionTok{mutate}\NormalTok{(}\AttributeTok{rank =} \FunctionTok{row\_number}\NormalTok{(),}
         \StringTok{\textasciigrave{}}\AttributeTok{term frequency}\StringTok{\textasciigrave{}} \OtherTok{=}\NormalTok{ n}\SpecialCharTok{/}\NormalTok{total)}
\CommentTok{\# head(train\_words\_sw)}
\FunctionTok{kable}\NormalTok{(train\_words\_sw[}\DecValTok{1}\SpecialCharTok{:}\DecValTok{10}\NormalTok{,], }\StringTok{"latex"}\NormalTok{, }\AttributeTok{escape=}\ConstantTok{FALSE}\NormalTok{, }\AttributeTok{booktabs=}\ConstantTok{TRUE}\NormalTok{, }\AttributeTok{linesep=}\StringTok{""}\NormalTok{, }\AttributeTok{caption=}\StringTok{"Frequency distribution of words excluding stopwords}\SpecialCharTok{\textbackslash{}\textbackslash{}}\StringTok{label\{tbl:train\_words\_sw\}"}\NormalTok{) }\CommentTok{\#\%\textgreater{}\%}
    \CommentTok{\# kable\_styling(latex\_options=c("HOLD\_position"), font\_size=6)}
\NormalTok{train\_words\_sw }\SpecialCharTok{\%\textgreater{}\%}
  \FunctionTok{top\_n}\NormalTok{(}\DecValTok{25}\NormalTok{, n) }\SpecialCharTok{\%\textgreater{}\%}
  \FunctionTok{ggplot}\NormalTok{(}\FunctionTok{aes}\NormalTok{(}\FunctionTok{reorder}\NormalTok{(word,n),n)) }\SpecialCharTok{+}
  \FunctionTok{geom\_col}\NormalTok{(}\AttributeTok{binwidth =} \DecValTok{1}\NormalTok{, }\AttributeTok{alpha =} \FloatTok{0.8}\NormalTok{) }\SpecialCharTok{+}
   \FunctionTok{coord\_flip}\NormalTok{() }\SpecialCharTok{+}
  \FunctionTok{labs}\NormalTok{(}\AttributeTok{y=}\StringTok{"n {-} Frequency distribution of words(ngrams) excluding stopwords"}\NormalTok{,}
       \AttributeTok{x=}\StringTok{"Top 25 words"}\NormalTok{)}
\CommentTok{\# 3. If the ratio is greater than 1500, tokenize the text as}
\CommentTok{\# sequences and use a sepCNN model}
\CommentTok{\#    see above}

\NormalTok{(S\_W\_ratio }\OtherTok{\textless{}{-}}\NormalTok{ num\_samples }\SpecialCharTok{/}\NormalTok{ median\_num\_words\_per\_sample)}
\NormalTok{amazon\_train }\SpecialCharTok{\%\textgreater{}\%}
  \FunctionTok{mutate}\NormalTok{(}\AttributeTok{n\_words =}\NormalTok{ tokenizers}\SpecialCharTok{::}\FunctionTok{count\_words}\NormalTok{(text)) }\SpecialCharTok{\%\textgreater{}\%}
  \FunctionTok{ggplot}\NormalTok{(}\FunctionTok{aes}\NormalTok{(n\_words)) }\SpecialCharTok{+}
  \FunctionTok{geom\_bar}\NormalTok{() }\SpecialCharTok{+}
  \FunctionTok{labs}\NormalTok{(}\AttributeTok{x =} \StringTok{"Number of words per review text"}\NormalTok{,}
       \AttributeTok{y =} \StringTok{"Number of review texts"}\NormalTok{)}
\NormalTok{amazon\_train }\SpecialCharTok{\%\textgreater{}\%}
  \FunctionTok{mutate}\NormalTok{(}\AttributeTok{n\_words =}\NormalTok{ tokenizers}\SpecialCharTok{::}\FunctionTok{count\_words}\NormalTok{(title)) }\SpecialCharTok{\%\textgreater{}\%}
  \FunctionTok{ggplot}\NormalTok{(}\FunctionTok{aes}\NormalTok{(n\_words)) }\SpecialCharTok{+}
  \FunctionTok{geom\_bar}\NormalTok{() }\SpecialCharTok{+}
  \FunctionTok{labs}\NormalTok{(}\AttributeTok{x =} \StringTok{"Number of words per review title"}\NormalTok{,}
       \AttributeTok{y =} \StringTok{"Number of review titles"}\NormalTok{)}
\NormalTok{amazon\_train }\SpecialCharTok{\%\textgreater{}\%}
  \FunctionTok{group\_by}\NormalTok{(label) }\SpecialCharTok{\%\textgreater{}\%}
  \FunctionTok{mutate}\NormalTok{(}\AttributeTok{n\_words =}\NormalTok{ tokenizers}\SpecialCharTok{::}\FunctionTok{count\_words}\NormalTok{(text)) }\SpecialCharTok{\%\textgreater{}\%}
  \FunctionTok{ggplot}\NormalTok{(}\FunctionTok{aes}\NormalTok{(n\_words)) }\SpecialCharTok{+}
  \CommentTok{\# ggplot(aes(nchar(text))) +}
  \FunctionTok{geom\_histogram}\NormalTok{(}\AttributeTok{binwidth =} \DecValTok{1}\NormalTok{, }\AttributeTok{alpha =} \FloatTok{0.8}\NormalTok{) }\SpecialCharTok{+}
  \FunctionTok{facet\_wrap}\NormalTok{(}\SpecialCharTok{\textasciitilde{}}\NormalTok{ label, }\AttributeTok{nrow =} \DecValTok{1}\NormalTok{) }\SpecialCharTok{+}
  \FunctionTok{labs}\NormalTok{(}\AttributeTok{x =} \StringTok{"Number of words per review text by label"}\NormalTok{,}
       \AttributeTok{y =} \StringTok{"Number of reviews"}\NormalTok{)}
\NormalTok{amazon\_train }\SpecialCharTok{\%\textgreater{}\%}
  \FunctionTok{group\_by}\NormalTok{(label) }\SpecialCharTok{\%\textgreater{}\%}
  \FunctionTok{mutate}\NormalTok{(}\AttributeTok{n\_words =}\NormalTok{ tokenizers}\SpecialCharTok{::}\FunctionTok{count\_words}\NormalTok{(title)) }\SpecialCharTok{\%\textgreater{}\%}
  \FunctionTok{ggplot}\NormalTok{(}\FunctionTok{aes}\NormalTok{(n\_words)) }\SpecialCharTok{+}
  \CommentTok{\# ggplot(aes(nchar(title))) +}
  \FunctionTok{geom\_histogram}\NormalTok{(}\AttributeTok{binwidth =} \DecValTok{1}\NormalTok{, }\AttributeTok{alpha =} \FloatTok{0.8}\NormalTok{) }\SpecialCharTok{+}
  \FunctionTok{facet\_wrap}\NormalTok{(}\SpecialCharTok{\textasciitilde{}}\NormalTok{ label, }\AttributeTok{nrow =} \DecValTok{1}\NormalTok{) }\SpecialCharTok{+}
  \FunctionTok{labs}\NormalTok{(}\AttributeTok{x =} \StringTok{"Number of words per review title by label"}\NormalTok{,}
       \AttributeTok{y =} \StringTok{"Number of reviews"}\NormalTok{)}
\NormalTok{amazon\_subset\_train }\OtherTok{\textless{}{-}}\NormalTok{ amazon\_train }\SpecialCharTok{\%\textgreater{}\%} \FunctionTok{select}\NormalTok{(}\SpecialCharTok{{-}}\NormalTok{title) }\SpecialCharTok{\%\textgreater{}\%}
  \FunctionTok{mutate}\NormalTok{(}\AttributeTok{n\_words =}\NormalTok{ tokenizers}\SpecialCharTok{::}\FunctionTok{count\_words}\NormalTok{(text)) }\SpecialCharTok{\%\textgreater{}\%}
  \FunctionTok{filter}\NormalTok{((n\_words }\SpecialCharTok{\textless{}} \DecValTok{35}\NormalTok{) }\SpecialCharTok{\&}\NormalTok{ (n\_words }\SpecialCharTok{\textgreater{}} \DecValTok{5}\NormalTok{)) }\SpecialCharTok{\%\textgreater{}\%} \FunctionTok{select}\NormalTok{(}\SpecialCharTok{{-}}\NormalTok{n\_words) }

\FunctionTok{dim}\NormalTok{(amazon\_subset\_train)}
\CommentTok{\# head(amazon\_subset\_train)}
\FunctionTok{kable}\NormalTok{(amazon\_subset\_train[}\DecValTok{1}\SpecialCharTok{:}\DecValTok{10}\NormalTok{,], }\StringTok{"latex"}\NormalTok{, }\AttributeTok{escape=}\ConstantTok{FALSE}\NormalTok{, }\AttributeTok{booktabs=}\ConstantTok{TRUE}\NormalTok{, }\AttributeTok{linesep=}\StringTok{""}\NormalTok{, }\AttributeTok{caption=}\StringTok{"Sample/Subset of our training dataset}\SpecialCharTok{\textbackslash{}\textbackslash{}}\StringTok{label\{tbl:amazon\_subset\_train\}"}\NormalTok{) }\CommentTok{\#\%\textgreater{}\%}
    \CommentTok{\# kable\_styling(latex\_options=c("HOLD\_position"), font\_size=6)}
\CommentTok{\# Free computer resources}
\FunctionTok{rm}\NormalTok{(amazon\_train, amazon\_val, amazon\_train\_text\_wordCount,num\_samples\_per\_class, temp, total\_words, train\_words)}
\FunctionTok{rm}\NormalTok{(mean\_num\_words\_per\_sample, median\_num\_words\_per\_sample, num\_classes, num\_samples, S\_W\_ratio)}
\FunctionTok{gc}\NormalTok{()}

\CommentTok{\# save(amazon\_subset\_train)}
\FunctionTok{write\_csv}\NormalTok{(amazon\_subset\_train,}\StringTok{"amazon\_review\_polarity\_csv/amazon\_subset\_train.csv"}\NormalTok{, }\AttributeTok{col\_names =} \ConstantTok{TRUE}\NormalTok{)}

\NormalTok{amazon\_train }\OtherTok{\textless{}{-}}\NormalTok{ amazon\_subset\_train}

\NormalTok{amazon\_train }\OtherTok{\textless{}{-}}\NormalTok{ amazon\_train }\SpecialCharTok{\%\textgreater{}\%}
  \FunctionTok{mutate}\NormalTok{(}\AttributeTok{label =} \FunctionTok{as.factor}\NormalTok{(label))}

\CommentTok{\# amazon\_val \textless{}{-} amazon\_train \%\textgreater{}\%}
\CommentTok{\#   mutate(label = as.factor(label))}
\FunctionTok{set.seed}\NormalTok{(}\DecValTok{1234}\NormalTok{)}

\NormalTok{amazon\_split }\OtherTok{\textless{}{-}}\NormalTok{ amazon\_train }\SpecialCharTok{\%\textgreater{}\%} \FunctionTok{initial\_split}\NormalTok{()}

\NormalTok{amazon\_train }\OtherTok{\textless{}{-}} \FunctionTok{training}\NormalTok{(amazon\_split)}
\NormalTok{amazon\_test }\OtherTok{\textless{}{-}} \FunctionTok{testing}\NormalTok{(amazon\_split)}

\FunctionTok{set.seed}\NormalTok{(}\DecValTok{123}\NormalTok{)}
\NormalTok{amazon\_folds }\OtherTok{\textless{}{-}} \FunctionTok{vfold\_cv}\NormalTok{(amazon\_train)}
\CommentTok{\# amazon\_folds}
\CommentTok{\# library(textrecipes)}

\NormalTok{amazon\_rec }\OtherTok{\textless{}{-}} \FunctionTok{recipe}\NormalTok{(label }\SpecialCharTok{\textasciitilde{}}\NormalTok{ text, }\AttributeTok{data =}\NormalTok{ amazon\_train) }\SpecialCharTok{\%\textgreater{}\%}
  \FunctionTok{step\_tokenize}\NormalTok{(text) }\SpecialCharTok{\%\textgreater{}\%}
  \FunctionTok{step\_tokenfilter}\NormalTok{(text, }\AttributeTok{max\_tokens =} \FloatTok{5e3}\NormalTok{) }\SpecialCharTok{\%\textgreater{}\%}
  \FunctionTok{step\_tfidf}\NormalTok{(text)}

\NormalTok{amazon\_rec}
\NormalTok{lasso\_spec }\OtherTok{\textless{}{-}} \FunctionTok{logistic\_reg}\NormalTok{(}\AttributeTok{penalty =} \FunctionTok{tune}\NormalTok{(), }\AttributeTok{mixture =} \DecValTok{1}\NormalTok{) }\SpecialCharTok{\%\textgreater{}\%}
  \FunctionTok{set\_mode}\NormalTok{(}\StringTok{"classification"}\NormalTok{) }\SpecialCharTok{\%\textgreater{}\%}
  \FunctionTok{set\_engine}\NormalTok{(}\StringTok{"glmnet"}\NormalTok{)}

\NormalTok{lasso\_spec}
\FunctionTok{library}\NormalTok{(hardhat)}
\NormalTok{sparse\_bp }\OtherTok{\textless{}{-}} \FunctionTok{default\_recipe\_blueprint}\NormalTok{(}\AttributeTok{composition =} \StringTok{"dgCMatrix"}\NormalTok{)}
\NormalTok{lambda\_grid }\OtherTok{\textless{}{-}} \FunctionTok{grid\_regular}\NormalTok{(}\FunctionTok{penalty}\NormalTok{(}\AttributeTok{range =} \FunctionTok{c}\NormalTok{(}\SpecialCharTok{{-}}\DecValTok{5}\NormalTok{, }\DecValTok{0}\NormalTok{)), }\AttributeTok{levels =} \DecValTok{20}\NormalTok{)}
\NormalTok{lambda\_grid}
\NormalTok{amazon\_wf }\OtherTok{\textless{}{-}} \FunctionTok{workflow}\NormalTok{() }\SpecialCharTok{\%\textgreater{}\%}
  \FunctionTok{add\_recipe}\NormalTok{(amazon\_rec, }\AttributeTok{blueprint =}\NormalTok{ sparse\_bp) }\SpecialCharTok{\%\textgreater{}\%}
  \FunctionTok{add\_model}\NormalTok{(lasso\_spec)}

\NormalTok{amazon\_wf}
\FunctionTok{set.seed}\NormalTok{(}\DecValTok{2020}\NormalTok{)}
\NormalTok{lasso\_rs }\OtherTok{\textless{}{-}} \FunctionTok{tune\_grid}\NormalTok{(}
\NormalTok{  amazon\_wf,}
\NormalTok{  amazon\_folds,}
  \AttributeTok{grid =}\NormalTok{ lambda\_grid,}
  \AttributeTok{control =} \FunctionTok{control\_resamples}\NormalTok{(}\AttributeTok{save\_pred =} \ConstantTok{TRUE}\NormalTok{)}
\NormalTok{)}

\CommentTok{\# lasso\_rs}
\NormalTok{m\_lm }\OtherTok{\textless{}{-}} \FunctionTok{collect\_metrics}\NormalTok{(lasso\_rs)}
\FunctionTok{kable}\NormalTok{(m\_lm, }\AttributeTok{format =} \StringTok{"simple"}\NormalTok{, }\AttributeTok{caption=}\StringTok{"Lasso Metrics}\SpecialCharTok{\textbackslash{}\textbackslash{}}\StringTok{label\{tbl:lasso\_metrics\}"}\NormalTok{)}
\NormalTok{m\_blr }\OtherTok{\textless{}{-}} \FunctionTok{show\_best}\NormalTok{(lasso\_rs, }\StringTok{"roc\_auc"}\NormalTok{)}
\FunctionTok{kable}\NormalTok{(m\_blr, }\AttributeTok{format =} \StringTok{"simple"}\NormalTok{, }\AttributeTok{caption=}\StringTok{"Best Lasso ROC}\SpecialCharTok{\textbackslash{}\textbackslash{}}\StringTok{label\{tbl:best\_lasso\_roc\}"}\NormalTok{)}
\NormalTok{m\_bla }\OtherTok{\textless{}{-}} \FunctionTok{show\_best}\NormalTok{(lasso\_rs, }\StringTok{"accuracy"}\NormalTok{)}
\FunctionTok{kable}\NormalTok{(m\_bla, }\AttributeTok{format =} \StringTok{"simple"}\NormalTok{, }\AttributeTok{caption=}\StringTok{"Best Lasso Accuracy}\SpecialCharTok{\textbackslash{}\textbackslash{}}\StringTok{label\{tbl:best\_lasso\_acc\}"}\NormalTok{)}
\FunctionTok{autoplot}\NormalTok{(lasso\_rs) }\SpecialCharTok{+}
  \FunctionTok{labs}\NormalTok{(}
    \AttributeTok{title =} \StringTok{"Lasso model performance across regularization penalties"}\NormalTok{,}
    \AttributeTok{subtitle =} \StringTok{"Performance metrics can be used to identify the best penalty"}
\NormalTok{  )}
\NormalTok{m\_lp }\OtherTok{\textless{}{-}} \FunctionTok{collect\_predictions}\NormalTok{(lasso\_rs)}
\FunctionTok{kable}\NormalTok{(}\FunctionTok{head}\NormalTok{(m\_lp), }\AttributeTok{format =} \StringTok{"simple"}\NormalTok{, }\AttributeTok{caption=}\StringTok{"Lasso Predictions}\SpecialCharTok{\textbackslash{}\textbackslash{}}\StringTok{label\{tbl:lasso\_predictions\}"}\NormalTok{)}
\NormalTok{m\_lp }\SpecialCharTok{\%\textgreater{}\%}
  \CommentTok{\# mutate(.pred\_class=as.numeric(levels(.pred\_class)[.pred\_class])) \%\textgreater{}\%}
  \FunctionTok{group\_by}\NormalTok{(id) }\SpecialCharTok{\%\textgreater{}\%}
  \FunctionTok{roc\_curve}\NormalTok{(}\AttributeTok{truth =}\NormalTok{ label, .pred\_0) }\SpecialCharTok{\%\textgreater{}\%}
  \FunctionTok{autoplot}\NormalTok{() }\SpecialCharTok{+}
  \FunctionTok{labs}\NormalTok{(}
    \AttributeTok{color =} \ConstantTok{NULL}\NormalTok{,}
    \AttributeTok{title =} \StringTok{"ROC curve for Lasso model Label 0"}\NormalTok{,}
    \AttributeTok{subtitle =} \StringTok{"Each resample fold is shown in a different color"}
\NormalTok{  )}
\NormalTok{m\_lp }\SpecialCharTok{\%\textgreater{}\%}
  \FunctionTok{group\_by}\NormalTok{(id) }\SpecialCharTok{\%\textgreater{}\%}
  \FunctionTok{roc\_curve}\NormalTok{(}\AttributeTok{truth =}\NormalTok{ label, .pred\_1) }\SpecialCharTok{\%\textgreater{}\%}
  \FunctionTok{autoplot}\NormalTok{() }\SpecialCharTok{+}
  \FunctionTok{labs}\NormalTok{(}
    \AttributeTok{color =} \ConstantTok{NULL}\NormalTok{,}
    \AttributeTok{title =} \StringTok{"ROC curve for Lasso model Label 1"}\NormalTok{,}
    \AttributeTok{subtitle =} \StringTok{"Each resample fold is shown in a different color"}
\NormalTok{  )}
\CommentTok{\# Best ROC\_AUC }
\NormalTok{blm\_best\_roc }\OtherTok{\textless{}{-}} \FunctionTok{max}\NormalTok{(m\_blr}\SpecialCharTok{$}\NormalTok{mean)  }

\CommentTok{\# Best Accuracy}
\NormalTok{blm\_best\_acc }\OtherTok{\textless{}{-}} \FunctionTok{max}\NormalTok{(m\_bla}\SpecialCharTok{$}\NormalTok{mean) }
\NormalTok{results\_table }\OtherTok{\textless{}{-}} \FunctionTok{tibble}\NormalTok{(}\AttributeTok{Index =} \StringTok{"1"}\NormalTok{, }\AttributeTok{Method =} \StringTok{"BLM"}\NormalTok{, }\AttributeTok{Accuracy =}\NormalTok{ blm\_best\_acc, }\AttributeTok{Loss =} \ConstantTok{NA}\NormalTok{)}

\FunctionTok{kable}\NormalTok{(results\_table, }\StringTok{"simple"}\NormalTok{,}\AttributeTok{caption=}\StringTok{"Baseline Linear Model Results}\SpecialCharTok{\textbackslash{}\textbackslash{}}\StringTok{label\{tbl:blm\_results\_table\}"}\NormalTok{)}
\CommentTok{\# \%\textgreater{}\%}
\CommentTok{\#     kable\_styling(latex\_options=c("HOLD\_position"), font\_size=7)}
\FunctionTok{rm}\NormalTok{(amazon\_folds, amazon\_rec, amazon\_split, amazon\_test, amazon\_train, amazon\_wf, lambda\_grid, lasso\_rs, lasso\_spec, sparse\_bp)}

\FunctionTok{gc}\NormalTok{()}

\NormalTok{amazon\_subset\_train }\OtherTok{\textless{}{-}}\NormalTok{ readr}\SpecialCharTok{::}\FunctionTok{read\_csv}\NormalTok{(}\StringTok{"amazon\_review\_polarity\_csv/amazon\_subset\_train.csv"}\NormalTok{)}

\NormalTok{amazon\_train }\OtherTok{\textless{}{-}}\NormalTok{ amazon\_subset\_train}

\NormalTok{max\_words }\OtherTok{\textless{}{-}} \FloatTok{2e4}
\NormalTok{max\_length }\OtherTok{\textless{}{-}} \DecValTok{30}
\NormalTok{mystopwords }\OtherTok{\textless{}{-}} \FunctionTok{c}\NormalTok{(}\StringTok{"s"}\NormalTok{, }\StringTok{"t"}\NormalTok{, }\StringTok{"m"}\NormalTok{, }\StringTok{"ve"}\NormalTok{, }\StringTok{"re"}\NormalTok{, }\StringTok{"d"}\NormalTok{, }\StringTok{"ll"}\NormalTok{)}

\NormalTok{amazon\_rec }\OtherTok{\textless{}{-}} \FunctionTok{recipe}\NormalTok{(}\SpecialCharTok{\textasciitilde{}}\NormalTok{ text, }\AttributeTok{data =}\NormalTok{ amazon\_subset\_train) }\SpecialCharTok{\%\textgreater{}\%}
  \FunctionTok{step\_text\_normalization}\NormalTok{(text) }\SpecialCharTok{\%\textgreater{}\%}
  \FunctionTok{step\_tokenize}\NormalTok{(text) }\SpecialCharTok{\%\textgreater{}\%}
  \FunctionTok{step\_stopwords}\NormalTok{(text, }
                 \AttributeTok{stopword\_source =} \StringTok{"stopwords{-}iso"}\NormalTok{,}
                 \AttributeTok{custom\_stopword\_source =}\NormalTok{ mystopwords) }\SpecialCharTok{\%\textgreater{}\%}
  \FunctionTok{step\_tokenfilter}\NormalTok{(text, }\AttributeTok{max\_tokens =}\NormalTok{ max\_words) }\SpecialCharTok{\%\textgreater{}\%}
  \FunctionTok{step\_sequence\_onehot}\NormalTok{(text, }\AttributeTok{sequence\_length =}\NormalTok{ max\_length)}

\NormalTok{amazon\_rec}
\NormalTok{amazon\_prep }\OtherTok{\textless{}{-}}  \FunctionTok{prep}\NormalTok{(amazon\_rec)}

\NormalTok{amazon\_subset\_train }\OtherTok{\textless{}{-}} \FunctionTok{bake}\NormalTok{(amazon\_prep, }\AttributeTok{new\_data =} \ConstantTok{NULL}\NormalTok{, }\AttributeTok{composition =} \StringTok{"matrix"}\NormalTok{)}
\FunctionTok{dim}\NormalTok{(amazon\_subset\_train)}
\NormalTok{amazon\_prep }\SpecialCharTok{\%\textgreater{}\%} \FunctionTok{tidy}\NormalTok{(}\DecValTok{5}\NormalTok{) }\SpecialCharTok{\%\textgreater{}\%} \FunctionTok{head}\NormalTok{(}\DecValTok{10}\NormalTok{)}
\CommentTok{\# library(keras)}
\CommentTok{\# use\_python(python = "/c/Users/bijoor/.conda/envs/tensorflow{-}python/python.exe", required = TRUE)}
\CommentTok{\# use\_condaenv(condaenv = "tensorflow{-}python", required = TRUE)}

\NormalTok{dense\_model }\OtherTok{\textless{}{-}} \FunctionTok{keras\_model\_sequential}\NormalTok{() }\SpecialCharTok{\%\textgreater{}\%}
  \FunctionTok{layer\_embedding}\NormalTok{(}\AttributeTok{input\_dim =}\NormalTok{ max\_words }\SpecialCharTok{+} \DecValTok{1}\NormalTok{,}
                  \AttributeTok{output\_dim =} \DecValTok{12}\NormalTok{,}
                  \AttributeTok{input\_length =}\NormalTok{ max\_length) }\SpecialCharTok{\%\textgreater{}\%}
  \FunctionTok{layer\_flatten}\NormalTok{() }\SpecialCharTok{\%\textgreater{}\%}
  \FunctionTok{layer\_layer\_normalization}\NormalTok{() }\SpecialCharTok{\%\textgreater{}\%}
  \CommentTok{\# layer\_dropout(0.1) \%\textgreater{}\%}
  \FunctionTok{layer\_dense}\NormalTok{(}\AttributeTok{units =} \DecValTok{64}\NormalTok{) }\SpecialCharTok{\%\textgreater{}\%}
  \CommentTok{\# layer\_activation\_leaky\_relu() \%\textgreater{}\%}
  \FunctionTok{layer\_activation\_relu}\NormalTok{() }\SpecialCharTok{\%\textgreater{}\%}
  \FunctionTok{layer\_dense}\NormalTok{(}\AttributeTok{units =} \DecValTok{1}\NormalTok{, }\AttributeTok{activation =} \StringTok{"sigmoid"}\NormalTok{)}

\NormalTok{dense\_model}
\CommentTok{\# opt \textless{}{-} optimizer\_adam(lr = 0.0001, decay = 1e{-}6)}
\CommentTok{\# opt \textless{}{-} optimizer\_sgd(lr = 0.001, decay = 1e{-}6)}
\NormalTok{opt }\OtherTok{\textless{}{-}} \FunctionTok{optimizer\_sgd}\NormalTok{()}
\NormalTok{dense\_model }\SpecialCharTok{\%\textgreater{}\%} \FunctionTok{compile}\NormalTok{(}
  \AttributeTok{optimizer =}\NormalTok{ opt,}
  \AttributeTok{loss =} \StringTok{"binary\_crossentropy"}\NormalTok{,}
  \AttributeTok{metrics =} \FunctionTok{c}\NormalTok{(}\StringTok{"accuracy"}\NormalTok{)}
\NormalTok{)}
\NormalTok{dense\_history }\OtherTok{\textless{}{-}}\NormalTok{ dense\_model }\SpecialCharTok{\%\textgreater{}\%}
  \FunctionTok{fit}\NormalTok{(}
    \AttributeTok{x =}\NormalTok{ amazon\_subset\_train,}
    \AttributeTok{y =}\NormalTok{ amazon\_train}\SpecialCharTok{$}\NormalTok{label,}
    \AttributeTok{batch\_size =} \DecValTok{1024}\NormalTok{,}
    \AttributeTok{epochs =} \DecValTok{50}\NormalTok{,}
    \AttributeTok{initial\_epoch =} \DecValTok{0}\NormalTok{,}
    \AttributeTok{validation\_split =} \FloatTok{0.20}\NormalTok{,}
    \AttributeTok{verbose =} \DecValTok{2}
\NormalTok{  )}

\NormalTok{dense\_history}

\FunctionTok{plot}\NormalTok{(dense\_history)}
\FunctionTok{set.seed}\NormalTok{(}\DecValTok{234}\NormalTok{)}
\NormalTok{amazon\_val\_eval }\OtherTok{\textless{}{-}} \FunctionTok{validation\_split}\NormalTok{(amazon\_train, }\AttributeTok{strata =}\NormalTok{ label)}
\CommentTok{\# amazon\_val\_eval \textless{}\textless{} I am getting a pandoc stack error printing this}
\NormalTok{amazon\_analysis }\OtherTok{\textless{}{-}} \FunctionTok{bake}\NormalTok{(amazon\_prep, }\AttributeTok{new\_data =} \FunctionTok{analysis}\NormalTok{(amazon\_val\_eval}\SpecialCharTok{$}\NormalTok{splits[[}\DecValTok{1}\NormalTok{]]),}
                        \AttributeTok{composition =} \StringTok{"matrix"}\NormalTok{)}
\FunctionTok{dim}\NormalTok{(amazon\_analysis)}
\NormalTok{amazon\_assess }\OtherTok{\textless{}{-}} \FunctionTok{bake}\NormalTok{(amazon\_prep, }\AttributeTok{new\_data =} \FunctionTok{assessment}\NormalTok{(amazon\_val\_eval}\SpecialCharTok{$}\NormalTok{splits[[}\DecValTok{1}\NormalTok{]]),}
                      \AttributeTok{composition =} \StringTok{"matrix"}\NormalTok{)}
\FunctionTok{dim}\NormalTok{(amazon\_assess)}
\NormalTok{label\_analysis }\OtherTok{\textless{}{-}} \FunctionTok{analysis}\NormalTok{(amazon\_val\_eval}\SpecialCharTok{$}\NormalTok{splits[[}\DecValTok{1}\NormalTok{]]) }\SpecialCharTok{\%\textgreater{}\%} \FunctionTok{pull}\NormalTok{(label)}
\NormalTok{label\_assess }\OtherTok{\textless{}{-}} \FunctionTok{assessment}\NormalTok{(amazon\_val\_eval}\SpecialCharTok{$}\NormalTok{splits[[}\DecValTok{1}\NormalTok{]]) }\SpecialCharTok{\%\textgreater{}\%} \FunctionTok{pull}\NormalTok{(label)}
\NormalTok{dense\_model }\OtherTok{\textless{}{-}} \FunctionTok{keras\_model\_sequential}\NormalTok{() }\SpecialCharTok{\%\textgreater{}\%}
  \FunctionTok{layer\_embedding}\NormalTok{(}\AttributeTok{input\_dim =}\NormalTok{ max\_words }\SpecialCharTok{+} \DecValTok{1}\NormalTok{,}
                  \AttributeTok{output\_dim =} \DecValTok{12}\NormalTok{,}
                  \AttributeTok{input\_length =}\NormalTok{ max\_length) }\SpecialCharTok{\%\textgreater{}\%}
  \FunctionTok{layer\_flatten}\NormalTok{() }\SpecialCharTok{\%\textgreater{}\%}
  \FunctionTok{layer\_layer\_normalization}\NormalTok{() }\SpecialCharTok{\%\textgreater{}\%}
  \FunctionTok{layer\_dropout}\NormalTok{(}\FloatTok{0.5}\NormalTok{) }\SpecialCharTok{\%\textgreater{}\%}
  \FunctionTok{layer\_dense}\NormalTok{(}\AttributeTok{units =} \DecValTok{64}\NormalTok{) }\SpecialCharTok{\%\textgreater{}\%}
  \FunctionTok{layer\_activation\_relu}\NormalTok{() }\SpecialCharTok{\%\textgreater{}\%}
  \FunctionTok{layer\_dropout}\NormalTok{(}\FloatTok{0.5}\NormalTok{) }\SpecialCharTok{\%\textgreater{}\%}
  \FunctionTok{layer\_dense}\NormalTok{(}\AttributeTok{units =} \DecValTok{128}\NormalTok{) }\SpecialCharTok{\%\textgreater{}\%}
  \FunctionTok{layer\_activation\_relu}\NormalTok{() }\SpecialCharTok{\%\textgreater{}\%}
  \FunctionTok{layer\_dense}\NormalTok{(}\AttributeTok{units =} \DecValTok{128}\NormalTok{) }\SpecialCharTok{\%\textgreater{}\%}
  \FunctionTok{layer\_activation\_relu}\NormalTok{() }\SpecialCharTok{\%\textgreater{}\%}
  \FunctionTok{layer\_dense}\NormalTok{(}\AttributeTok{units =} \DecValTok{1}\NormalTok{, }\AttributeTok{activation =} \StringTok{"sigmoid"}\NormalTok{)}

\NormalTok{opt }\OtherTok{\textless{}{-}} \FunctionTok{optimizer\_adam}\NormalTok{(}\AttributeTok{lr =} \FloatTok{0.0001}\NormalTok{, }\AttributeTok{decay =} \FloatTok{1e{-}6}\NormalTok{)}
\CommentTok{\# opt \textless{}{-} optimizer\_sgd(lr = 0.001, decay = 1e{-}6)}
\CommentTok{\# opt \textless{}{-} optimizer\_sgd()}
\NormalTok{dense\_model }\SpecialCharTok{\%\textgreater{}\%} \FunctionTok{compile}\NormalTok{(}
  \AttributeTok{optimizer =}\NormalTok{ opt,}
  \AttributeTok{loss =} \StringTok{"binary\_crossentropy"}\NormalTok{,}
  \AttributeTok{metrics =} \FunctionTok{c}\NormalTok{(}\StringTok{"accuracy"}\NormalTok{)}
\NormalTok{)}
\NormalTok{val\_history }\OtherTok{\textless{}{-}}\NormalTok{ dense\_model }\SpecialCharTok{\%\textgreater{}\%}
  \FunctionTok{fit}\NormalTok{(}
    \AttributeTok{x =}\NormalTok{ amazon\_analysis,}
    \AttributeTok{y =}\NormalTok{ label\_analysis,}
    \AttributeTok{batch\_size =} \DecValTok{2048}\NormalTok{,}
    \AttributeTok{epochs =} \DecValTok{20}\NormalTok{,}
    \AttributeTok{validation\_data =} \FunctionTok{list}\NormalTok{(amazon\_assess, label\_assess),}
    \AttributeTok{verbose =} \DecValTok{2}
\NormalTok{  )}

\NormalTok{val\_history}
\FunctionTok{plot}\NormalTok{(val\_history)}
\NormalTok{keras\_predict }\OtherTok{\textless{}{-}} \ControlFlowTok{function}\NormalTok{(model, baked\_data, response) \{}
\NormalTok{  predictions }\OtherTok{\textless{}{-}} \FunctionTok{predict}\NormalTok{(model, baked\_data)[, }\DecValTok{1}\NormalTok{]}
  \FunctionTok{tibble}\NormalTok{(}
    \AttributeTok{.pred\_1 =}\NormalTok{ predictions,}
    \AttributeTok{.pred\_class =} \FunctionTok{if\_else}\NormalTok{(.pred\_1 }\SpecialCharTok{\textless{}} \FloatTok{0.5}\NormalTok{, }\DecValTok{0}\NormalTok{, }\DecValTok{1}\NormalTok{),}
    \AttributeTok{label =}\NormalTok{ response) }\SpecialCharTok{\%\textgreater{}\%} 
    \FunctionTok{mutate}\NormalTok{(}\FunctionTok{across}\NormalTok{(}\FunctionTok{c}\NormalTok{(label, .pred\_class), }
                  \SpecialCharTok{\textasciitilde{}} \FunctionTok{factor}\NormalTok{(.x, }\AttributeTok{levels =} \FunctionTok{c}\NormalTok{(}\DecValTok{1}\NormalTok{, }\DecValTok{0}\NormalTok{))))}
\NormalTok{\}}
\NormalTok{val\_res }\OtherTok{\textless{}{-}} \FunctionTok{keras\_predict}\NormalTok{(dense\_model, amazon\_assess, label\_assess)}
\CommentTok{\# head(val\_res)}
\FunctionTok{kable}\NormalTok{(}\FunctionTok{head}\NormalTok{(val\_res), }\AttributeTok{format=}\StringTok{"simple"}\NormalTok{, }\AttributeTok{caption=}\StringTok{"DNN Model 2 Predictions using validation data}\SpecialCharTok{\textbackslash{}\textbackslash{}}\StringTok{label\{tbl:val\_res\}"}\NormalTok{)}
\NormalTok{m1 }\OtherTok{\textless{}{-}} \FunctionTok{metrics}\NormalTok{(val\_res, label, .pred\_class)}
\FunctionTok{kable}\NormalTok{(m1, }\AttributeTok{format =} \StringTok{"simple"}\NormalTok{, }\AttributeTok{caption=}\StringTok{"DNN Model 2 Metrics using Validation data}\SpecialCharTok{\textbackslash{}\textbackslash{}}\StringTok{label\{tbl:val\_res\_metrics\}"}\NormalTok{)}
\NormalTok{val\_res }\SpecialCharTok{\%\textgreater{}\%}
  \FunctionTok{conf\_mat}\NormalTok{(label, .pred\_class) }\SpecialCharTok{\%\textgreater{}\%}
  \FunctionTok{autoplot}\NormalTok{(}\AttributeTok{type =} \StringTok{"heatmap"}\NormalTok{)}
\NormalTok{val\_res }\SpecialCharTok{\%\textgreater{}\%}
  \FunctionTok{roc\_curve}\NormalTok{(}\AttributeTok{truth =}\NormalTok{ label, .pred\_1) }\SpecialCharTok{\%\textgreater{}\%}
  \FunctionTok{autoplot}\NormalTok{() }\SpecialCharTok{+}
  \FunctionTok{labs}\NormalTok{(}
    \AttributeTok{title =} \StringTok{"Receiver operator curve for Amazon Reviews"}
\NormalTok{  )}
\CommentTok{\# Best DNN accuracy}
\NormalTok{dnn\_best\_acc }\OtherTok{\textless{}{-}} \FunctionTok{max}\NormalTok{(val\_history}\SpecialCharTok{$}\NormalTok{metrics}\SpecialCharTok{$}\NormalTok{val\_accuracy) }

\CommentTok{\# Lowest DNN Loss}
\NormalTok{dnn\_lowest\_loss }\OtherTok{\textless{}{-}} \FunctionTok{min}\NormalTok{(val\_history}\SpecialCharTok{$}\NormalTok{metrics}\SpecialCharTok{$}\NormalTok{val\_loss)}
\NormalTok{results\_table }\OtherTok{\textless{}{-}} \FunctionTok{bind\_rows}\NormalTok{(results\_table,}
                           \FunctionTok{tibble}\NormalTok{(}\AttributeTok{Index =} \StringTok{"2"}\NormalTok{,}
                                  \AttributeTok{Method =} \StringTok{"DNN"}\NormalTok{,}
                                  \AttributeTok{Accuracy =}\NormalTok{ dnn\_best\_acc,}
                                  \AttributeTok{Loss =}\NormalTok{ dnn\_lowest\_loss))}

\FunctionTok{kable}\NormalTok{(results\_table, }\StringTok{"simple"}\NormalTok{,}\AttributeTok{caption=}\StringTok{"DNN Model Results}\SpecialCharTok{\textbackslash{}\textbackslash{}}\StringTok{label\{tbl:dnn\_results\_table\}"}\NormalTok{)}
\CommentTok{\# \%\textgreater{}\%}
\CommentTok{\#     kable\_styling(latex\_options=c("HOLD\_position"), font\_size=7)}
\NormalTok{simple\_cnn\_model }\OtherTok{\textless{}{-}} \FunctionTok{keras\_model\_sequential}\NormalTok{() }\SpecialCharTok{\%\textgreater{}\%}
  \FunctionTok{layer\_embedding}\NormalTok{(}\AttributeTok{input\_dim =}\NormalTok{ max\_words }\SpecialCharTok{+} \DecValTok{1}\NormalTok{, }\AttributeTok{output\_dim =} \DecValTok{16}\NormalTok{,}
                  \AttributeTok{input\_length =}\NormalTok{ max\_length) }\SpecialCharTok{\%\textgreater{}\%}
  \FunctionTok{layer\_batch\_normalization}\NormalTok{() }\SpecialCharTok{\%\textgreater{}\%}
  \FunctionTok{layer\_conv\_1d}\NormalTok{(}\AttributeTok{filter =} \DecValTok{32}\NormalTok{, }\AttributeTok{kernel\_size =} \DecValTok{5}\NormalTok{, }\AttributeTok{activation =} \StringTok{"relu"}\NormalTok{) }\SpecialCharTok{\%\textgreater{}\%}
  \FunctionTok{layer\_max\_pooling\_1d}\NormalTok{(}\AttributeTok{pool\_size =} \DecValTok{2}\NormalTok{) }\SpecialCharTok{\%\textgreater{}\%}
  \FunctionTok{layer\_conv\_1d}\NormalTok{(}\AttributeTok{filter =} \DecValTok{64}\NormalTok{, }\AttributeTok{kernel\_size =} \DecValTok{3}\NormalTok{, }\AttributeTok{activation =} \StringTok{"relu"}\NormalTok{) }\SpecialCharTok{\%\textgreater{}\%}
  \FunctionTok{layer\_global\_max\_pooling\_1d}\NormalTok{() }\SpecialCharTok{\%\textgreater{}\%} 
  \FunctionTok{layer\_dense}\NormalTok{(}\AttributeTok{units =} \DecValTok{64}\NormalTok{, }\AttributeTok{activation =} \StringTok{"relu"}\NormalTok{) }\SpecialCharTok{\%\textgreater{}\%}
  \FunctionTok{layer\_dense}\NormalTok{(}\AttributeTok{units =} \DecValTok{1}\NormalTok{, }\AttributeTok{activation =} \StringTok{"sigmoid"}\NormalTok{)}

\NormalTok{simple\_cnn\_model}
\CommentTok{\# opt \textless{}{-} optimizer\_sgd(lr = 0.001, decay = 1e{-}6)}
\CommentTok{\# opt \textless{}{-} optimizer\_adam()}
\CommentTok{\# opt \textless{}{-} optimizer\_sgd()}
\NormalTok{opt }\OtherTok{\textless{}{-}} \FunctionTok{optimizer\_adam}\NormalTok{(}\AttributeTok{lr =} \FloatTok{0.0001}\NormalTok{, }\AttributeTok{decay =} \FloatTok{1e{-}6}\NormalTok{)}
\NormalTok{simple\_cnn\_model }\SpecialCharTok{\%\textgreater{}\%} \FunctionTok{compile}\NormalTok{(}
  \AttributeTok{optimizer =}\NormalTok{ opt,}
  \AttributeTok{loss =} \StringTok{"binary\_crossentropy"}\NormalTok{,}
  \AttributeTok{metrics =} \FunctionTok{c}\NormalTok{(}\StringTok{"accuracy"}\NormalTok{)}
\NormalTok{)}
\NormalTok{simple\_cnn\_val\_history }\OtherTok{\textless{}{-}}\NormalTok{ simple\_cnn\_model }\SpecialCharTok{\%\textgreater{}\%}
  \FunctionTok{fit}\NormalTok{(}
    \AttributeTok{x =}\NormalTok{ amazon\_analysis,}
    \AttributeTok{y =}\NormalTok{ label\_analysis,}
    \AttributeTok{batch\_size =} \DecValTok{1024}\NormalTok{,}
    \AttributeTok{epochs =} \DecValTok{7}\NormalTok{,}
    \AttributeTok{initial\_epoch =} \DecValTok{0}\NormalTok{,}
    \AttributeTok{validation\_data =} \FunctionTok{list}\NormalTok{(amazon\_assess, label\_assess),}
    \AttributeTok{verbose =} \DecValTok{2}
\NormalTok{  )}

\NormalTok{simple\_cnn\_val\_history}
\FunctionTok{plot}\NormalTok{(simple\_cnn\_val\_history)}
\NormalTok{simple\_cnn\_val\_res }\OtherTok{\textless{}{-}} \FunctionTok{keras\_predict}\NormalTok{(simple\_cnn\_model, amazon\_assess, label\_assess)}
\CommentTok{\# head(simple\_cnn\_val\_res)}
\FunctionTok{kable}\NormalTok{(}\FunctionTok{head}\NormalTok{(simple\_cnn\_val\_res), }\AttributeTok{format=}\StringTok{"simple"}\NormalTok{, }\AttributeTok{caption=}\StringTok{"CNN Model Predictions using validation data}\SpecialCharTok{\textbackslash{}\textbackslash{}}\StringTok{label\{tbl:simple\_cnn\_val\_res\}"}\NormalTok{)}
\NormalTok{m2 }\OtherTok{\textless{}{-}} \FunctionTok{metrics}\NormalTok{(simple\_cnn\_val\_res, label, .pred\_class)}
\FunctionTok{kable}\NormalTok{(m2, }\AttributeTok{format=}\StringTok{"simple"}\NormalTok{, }\AttributeTok{caption=}\StringTok{"CNN Model Metrics using validation data}\SpecialCharTok{\textbackslash{}\textbackslash{}}\StringTok{label\{tbl:simple\_cnn\_val\_res\_metrics\}"}\NormalTok{)}
\NormalTok{simple\_cnn\_val\_res }\SpecialCharTok{\%\textgreater{}\%}
  \FunctionTok{conf\_mat}\NormalTok{(label, .pred\_class) }\SpecialCharTok{\%\textgreater{}\%}
  \FunctionTok{autoplot}\NormalTok{(}\AttributeTok{type =} \StringTok{"heatmap"}\NormalTok{)}
\NormalTok{simple\_cnn\_val\_res }\SpecialCharTok{\%\textgreater{}\%}
  \FunctionTok{roc\_curve}\NormalTok{(}\AttributeTok{truth =}\NormalTok{ label, .pred\_1) }\SpecialCharTok{\%\textgreater{}\%}
  \FunctionTok{autoplot}\NormalTok{() }\SpecialCharTok{+}
  \FunctionTok{labs}\NormalTok{(}
    \AttributeTok{title =} \StringTok{"Receiver operator curve for Amazon Reviews"}
\NormalTok{  )}
\CommentTok{\# Best CNN accuracy}
\NormalTok{cnn\_best\_acc }\OtherTok{\textless{}{-}} \FunctionTok{max}\NormalTok{(simple\_cnn\_val\_history}\SpecialCharTok{$}\NormalTok{metrics}\SpecialCharTok{$}\NormalTok{val\_accuracy) }

\CommentTok{\# Lowest CNN Loss}
\NormalTok{cnn\_lowest\_loss }\OtherTok{\textless{}{-}} \FunctionTok{min}\NormalTok{(simple\_cnn\_val\_history}\SpecialCharTok{$}\NormalTok{metrics}\SpecialCharTok{$}\NormalTok{val\_loss)}
\NormalTok{results\_table }\OtherTok{\textless{}{-}} \FunctionTok{bind\_rows}\NormalTok{(results\_table,}
                           \FunctionTok{tibble}\NormalTok{(}\AttributeTok{Index =} \StringTok{"3"}\NormalTok{,}
                                  \AttributeTok{Method =} \StringTok{"CNN"}\NormalTok{,}
                                  \AttributeTok{Accuracy =}\NormalTok{ cnn\_best\_acc,}
                                  \AttributeTok{Loss =}\NormalTok{ cnn\_lowest\_loss))}

\FunctionTok{kable}\NormalTok{(results\_table, }\StringTok{"simple"}\NormalTok{,}\AttributeTok{caption=}\StringTok{"CNN Model Results}\SpecialCharTok{\textbackslash{}\textbackslash{}}\StringTok{label\{tbl:cnn\_results\_table\}"}\NormalTok{)}
\CommentTok{\# \%\textgreater{}\%}
\CommentTok{\#     kable\_styling(latex\_options=c("HOLD\_position"), font\_size=7)}
\NormalTok{sep\_cnn\_model }\OtherTok{\textless{}{-}} \FunctionTok{keras\_model\_sequential}\NormalTok{() }\SpecialCharTok{\%\textgreater{}\%}
  \FunctionTok{layer\_embedding}\NormalTok{(}\AttributeTok{input\_dim =}\NormalTok{ max\_words }\SpecialCharTok{+} \DecValTok{1}\NormalTok{, }\AttributeTok{output\_dim =} \DecValTok{16}\NormalTok{,}
                  \AttributeTok{input\_length =}\NormalTok{ max\_length) }\SpecialCharTok{\%\textgreater{}\%}
  \CommentTok{\# layer\_batch\_normalization() \%\textgreater{}\%}
  \FunctionTok{layer\_dropout}\NormalTok{(}\FloatTok{0.2}\NormalTok{) }\SpecialCharTok{\%\textgreater{}\%}
  \FunctionTok{layer\_separable\_conv\_1d}\NormalTok{(}\AttributeTok{filter =} \DecValTok{32}\NormalTok{, }\AttributeTok{kernel\_size =} \DecValTok{5}\NormalTok{, }\AttributeTok{activation =} \StringTok{"relu"}\NormalTok{) }\SpecialCharTok{\%\textgreater{}\%}
  \FunctionTok{layer\_separable\_conv\_1d}\NormalTok{(}\AttributeTok{filter =} \DecValTok{32}\NormalTok{, }\AttributeTok{kernel\_size =} \DecValTok{5}\NormalTok{, }\AttributeTok{activation =} \StringTok{"relu"}\NormalTok{) }\SpecialCharTok{\%\textgreater{}\%}
  \FunctionTok{layer\_max\_pooling\_1d}\NormalTok{(}\AttributeTok{pool\_size =} \DecValTok{2}\NormalTok{) }\SpecialCharTok{\%\textgreater{}\%}
  \FunctionTok{layer\_separable\_conv\_1d}\NormalTok{(}\AttributeTok{filter =} \DecValTok{64}\NormalTok{, }\AttributeTok{kernel\_size =} \DecValTok{5}\NormalTok{, }\AttributeTok{activation =} \StringTok{"relu"}\NormalTok{) }\SpecialCharTok{\%\textgreater{}\%}
  \FunctionTok{layer\_separable\_conv\_1d}\NormalTok{(}\AttributeTok{filter =} \DecValTok{64}\NormalTok{, }\AttributeTok{kernel\_size =} \DecValTok{5}\NormalTok{, }\AttributeTok{activation =} \StringTok{"relu"}\NormalTok{) }\SpecialCharTok{\%\textgreater{}\%}
  \FunctionTok{layer\_global\_average\_pooling\_1d}\NormalTok{() }\SpecialCharTok{\%\textgreater{}\%} 
  \FunctionTok{layer\_dropout}\NormalTok{(}\FloatTok{0.2}\NormalTok{) }\SpecialCharTok{\%\textgreater{}\%}
  \CommentTok{\# layer\_dense(units = 64, activation = "relu") \%\textgreater{}\%}
  \FunctionTok{layer\_dense}\NormalTok{(}\AttributeTok{units =} \DecValTok{1}\NormalTok{, }\AttributeTok{activation =} \StringTok{"sigmoid"}\NormalTok{)}

\NormalTok{sep\_cnn\_model}
\CommentTok{\# opt \textless{}{-} optimizer\_sgd(lr = 0.001, decay = 1e{-}6)}
\CommentTok{\# opt \textless{}{-} optimizer\_adam()}
\CommentTok{\# opt \textless{}{-} optimizer\_sgd()}
\NormalTok{opt }\OtherTok{\textless{}{-}} \FunctionTok{optimizer\_adam}\NormalTok{(}\AttributeTok{lr =} \FloatTok{0.0001}\NormalTok{, }\AttributeTok{decay =} \FloatTok{1e{-}6}\NormalTok{)}
\NormalTok{sep\_cnn\_model }\SpecialCharTok{\%\textgreater{}\%} \FunctionTok{compile}\NormalTok{(}
  \AttributeTok{optimizer =}\NormalTok{ opt,}
  \AttributeTok{loss =} \StringTok{"binary\_crossentropy"}\NormalTok{,}
  \AttributeTok{metrics =} \FunctionTok{c}\NormalTok{(}\StringTok{"accuracy"}\NormalTok{)}
\NormalTok{)}
\NormalTok{sep\_cnn\_val\_history }\OtherTok{\textless{}{-}}\NormalTok{ sep\_cnn\_model }\SpecialCharTok{\%\textgreater{}\%}
  \FunctionTok{fit}\NormalTok{(}
    \AttributeTok{x =}\NormalTok{ amazon\_analysis,}
    \AttributeTok{y =}\NormalTok{ label\_analysis,}
    \AttributeTok{batch\_size =} \DecValTok{128}\NormalTok{,}
    \AttributeTok{epochs =} \DecValTok{20}\NormalTok{,}
    \AttributeTok{initial\_epoch =} \DecValTok{0}\NormalTok{,}
    \AttributeTok{validation\_data =} \FunctionTok{list}\NormalTok{(amazon\_assess, label\_assess),}
    \AttributeTok{callbacks =} \FunctionTok{list}\NormalTok{(}\FunctionTok{callback\_early\_stopping}\NormalTok{(}
        \AttributeTok{monitor=}\StringTok{\textquotesingle{}val\_loss\textquotesingle{}}\NormalTok{, }\AttributeTok{patience=}\DecValTok{2}\NormalTok{)),}
    \AttributeTok{verbose =} \DecValTok{2}
\NormalTok{  )}

\NormalTok{sep\_cnn\_val\_history}
\FunctionTok{plot}\NormalTok{(sep\_cnn\_val\_history)}
\NormalTok{sep\_cnn\_val\_res }\OtherTok{\textless{}{-}} \FunctionTok{keras\_predict}\NormalTok{(sep\_cnn\_model, amazon\_assess, label\_assess)}
\CommentTok{\# head(sep\_cnn\_val\_res)}
\FunctionTok{kable}\NormalTok{(}\FunctionTok{head}\NormalTok{(sep\_cnn\_val\_res), }\AttributeTok{format=}\StringTok{"simple"}\NormalTok{, }\AttributeTok{caption=}\StringTok{"sepCNN Model Predictions using validation data}\SpecialCharTok{\textbackslash{}\textbackslash{}}\StringTok{label\{tbl:sep\_cnn\_val\_res\}"}\NormalTok{)}
\NormalTok{m3 }\OtherTok{\textless{}{-}} \FunctionTok{metrics}\NormalTok{(sep\_cnn\_val\_res, label, .pred\_class)}
\FunctionTok{kable}\NormalTok{(m3, }\AttributeTok{format=}\StringTok{"simple"}\NormalTok{, }\AttributeTok{caption=}\StringTok{"sepCNN Model Metrics using validation data}\SpecialCharTok{\textbackslash{}\textbackslash{}}\StringTok{label\{tbl:sep\_cnn\_val\_res\_metrics\}"}\NormalTok{)}
\NormalTok{sep\_cnn\_val\_res }\SpecialCharTok{\%\textgreater{}\%}
  \FunctionTok{conf\_mat}\NormalTok{(label, .pred\_class) }\SpecialCharTok{\%\textgreater{}\%}
  \FunctionTok{autoplot}\NormalTok{(}\AttributeTok{type =} \StringTok{"heatmap"}\NormalTok{)}
\NormalTok{sep\_cnn\_val\_res }\SpecialCharTok{\%\textgreater{}\%}
  \FunctionTok{roc\_curve}\NormalTok{(}\AttributeTok{truth =}\NormalTok{ label, .pred\_1) }\SpecialCharTok{\%\textgreater{}\%}
  \FunctionTok{autoplot}\NormalTok{() }\SpecialCharTok{+}
  \FunctionTok{labs}\NormalTok{(}
    \AttributeTok{title =} \StringTok{"Receiver operator curve for Amazon Reviews"}
\NormalTok{  )}
\CommentTok{\# Best sepCNN accuracy}
\NormalTok{sep\_cnn\_best\_acc }\OtherTok{\textless{}{-}} \FunctionTok{max}\NormalTok{(sep\_cnn\_val\_history}\SpecialCharTok{$}\NormalTok{metrics}\SpecialCharTok{$}\NormalTok{val\_accuracy) }

\CommentTok{\# Lowest sepCNN Loss}
\NormalTok{sep\_cnn\_lowest\_loss }\OtherTok{\textless{}{-}} \FunctionTok{min}\NormalTok{(sep\_cnn\_val\_history}\SpecialCharTok{$}\NormalTok{metrics}\SpecialCharTok{$}\NormalTok{val\_loss)}
\NormalTok{results\_table }\OtherTok{\textless{}{-}} \FunctionTok{bind\_rows}\NormalTok{(results\_table,}
                           \FunctionTok{tibble}\NormalTok{(}\AttributeTok{Index =} \StringTok{"4"}\NormalTok{,}
                                  \AttributeTok{Method =} \StringTok{"sepCNN"}\NormalTok{,}
                                  \AttributeTok{Accuracy =}\NormalTok{ sep\_cnn\_best\_acc,}
                                  \AttributeTok{Loss =}\NormalTok{ sep\_cnn\_lowest\_loss))}

\FunctionTok{kable}\NormalTok{(results\_table, }\StringTok{"simple"}\NormalTok{,}\AttributeTok{caption=}\StringTok{"sepCNN Model Results}\SpecialCharTok{\textbackslash{}\textbackslash{}}\StringTok{label\{tbl:sep\_cnn\_results\_table\}"}\NormalTok{)}
\CommentTok{\# \%\textgreater{}\%}
\CommentTok{\#     kable\_styling(latex\_options=c("HOLD\_position"), font\_size=7)}
\FunctionTok{library}\NormalTok{(keras)}
\FunctionTok{library}\NormalTok{(tfdatasets)}
\FunctionTok{library}\NormalTok{(reticulate)}
\FunctionTok{library}\NormalTok{(tidyverse)}
\FunctionTok{library}\NormalTok{(lubridate)}
\FunctionTok{library}\NormalTok{(tfhub)}

\CommentTok{\# A dependency of the preprocessing for BERT inputs}
\CommentTok{\# pip install {-}q {-}U tensorflow{-}text}
\FunctionTok{import}\NormalTok{(}\StringTok{"tensorflow\_text"}\NormalTok{)}

\CommentTok{\# You will use the AdamW optimizer from tensorflow/models.}
\CommentTok{\# pip install {-}q tf{-}models{-}official}
\CommentTok{\# to create AdamW optimizer}
\NormalTok{o\_nlp }\OtherTok{\textless{}{-}} \FunctionTok{import}\NormalTok{(}\StringTok{"official.nlp"}\NormalTok{)}

\FunctionTok{Sys.setenv}\NormalTok{(}\AttributeTok{TFHUB\_CACHE\_DIR=}\StringTok{"C:/Users/bijoor/.cache/tfhub\_modules"}\NormalTok{)}
\FunctionTok{Sys.getenv}\NormalTok{(}\StringTok{"TFHUB\_CACHE\_DIR"}\NormalTok{)}

\NormalTok{train\_file\_path }\OtherTok{\textless{}{-}} \FunctionTok{file.path}\NormalTok{(}\StringTok{"amazon\_review\_polarity\_csv/amazon\_train.csv"}\NormalTok{)}

\NormalTok{batch\_size }\OtherTok{\textless{}{-}} \DecValTok{32}

\NormalTok{train\_dataset }\OtherTok{\textless{}{-}} \FunctionTok{make\_csv\_dataset}\NormalTok{(}
\NormalTok{  train\_file\_path, }
  \AttributeTok{field\_delim =} \StringTok{","}\NormalTok{,}
  \AttributeTok{batch\_size =}\NormalTok{ batch\_size,}
  \AttributeTok{column\_names =} \FunctionTok{list}\NormalTok{(}\StringTok{"label"}\NormalTok{, }\StringTok{"title"}\NormalTok{, }\StringTok{"text"}\NormalTok{),}
  \AttributeTok{label\_name =} \StringTok{"label"}\NormalTok{,}
  \AttributeTok{select\_columns =} \FunctionTok{list}\NormalTok{(}\StringTok{"label"}\NormalTok{, }\StringTok{"text"}\NormalTok{),}
  \AttributeTok{num\_epochs =} \DecValTok{1}
\NormalTok{)}


\NormalTok{train\_dataset }\SpecialCharTok{\%\textgreater{}\%}
\NormalTok{  reticulate}\SpecialCharTok{::}\FunctionTok{as\_iterator}\NormalTok{() }\SpecialCharTok{\%\textgreater{}\%} 
\NormalTok{  reticulate}\SpecialCharTok{::}\FunctionTok{iter\_next}\NormalTok{() }\CommentTok{\#\%\textgreater{}\% }
  \CommentTok{\# reticulate::py\_to\_r()}

\CommentTok{\# {-}{-}{-}{-}{-}{-}{-}{-}{-}{-}{-}{-}{-}{-}{-}{-}{-}{-}{-}{-}{-}{-}}

\NormalTok{val\_file\_path }\OtherTok{\textless{}{-}} \FunctionTok{file.path}\NormalTok{(}\StringTok{"amazon\_review\_polarity\_csv/amazon\_val.csv"}\NormalTok{)}

\NormalTok{val\_dataset }\OtherTok{\textless{}{-}} \FunctionTok{make\_csv\_dataset}\NormalTok{(}
\NormalTok{  val\_file\_path, }
  \AttributeTok{field\_delim =} \StringTok{","}\NormalTok{,}
  \AttributeTok{batch\_size =}\NormalTok{ batch\_size,}
  \AttributeTok{column\_names =} \FunctionTok{list}\NormalTok{(}\StringTok{"label"}\NormalTok{, }\StringTok{"title"}\NormalTok{, }\StringTok{"text"}\NormalTok{),}
  \AttributeTok{label\_name =} \StringTok{"label"}\NormalTok{,}
  \AttributeTok{select\_columns =} \FunctionTok{list}\NormalTok{(}\StringTok{"label"}\NormalTok{, }\StringTok{"text"}\NormalTok{),}
  \AttributeTok{num\_epochs =} \DecValTok{1}
\NormalTok{)}


\NormalTok{val\_dataset }\SpecialCharTok{\%\textgreater{}\%}
\NormalTok{  reticulate}\SpecialCharTok{::}\FunctionTok{as\_iterator}\NormalTok{() }\SpecialCharTok{\%\textgreater{}\%} 
\NormalTok{  reticulate}\SpecialCharTok{::}\FunctionTok{iter\_next}\NormalTok{()}

\CommentTok{\# {-}{-}{-}{-}{-}{-}{-}{-}{-}{-}{-}{-}{-}{-}{-}{-}{-}{-}{-}{-}{-}{-}{-}{-}{-}{-}{-}{-}{-}{-}{-}{-}{-}{-}{-}}

\NormalTok{test\_file\_path }\OtherTok{\textless{}{-}} \FunctionTok{file.path}\NormalTok{(}\StringTok{"amazon\_review\_polarity\_csv/amazon\_test.csv"}\NormalTok{)}


\NormalTok{test\_dataset }\OtherTok{\textless{}{-}} \FunctionTok{make\_csv\_dataset}\NormalTok{(}
\NormalTok{  test\_file\_path, }
  \AttributeTok{field\_delim =} \StringTok{","}\NormalTok{,}
  \AttributeTok{batch\_size =}\NormalTok{ batch\_size,}
  \AttributeTok{column\_names =} \FunctionTok{list}\NormalTok{(}\StringTok{"label"}\NormalTok{, }\StringTok{"title"}\NormalTok{, }\StringTok{"text"}\NormalTok{),}
  \AttributeTok{label\_name =} \StringTok{"label"}\NormalTok{,}
  \AttributeTok{select\_columns =} \FunctionTok{list}\NormalTok{(}\StringTok{"label"}\NormalTok{, }\StringTok{"text"}\NormalTok{),}
  \AttributeTok{num\_epochs =} \DecValTok{1}
\NormalTok{)}

\NormalTok{test\_dataset }\SpecialCharTok{\%\textgreater{}\%}
\NormalTok{  reticulate}\SpecialCharTok{::}\FunctionTok{as\_iterator}\NormalTok{() }\SpecialCharTok{\%\textgreater{}\%} 
\NormalTok{  reticulate}\SpecialCharTok{::}\FunctionTok{iter\_next}\NormalTok{()}

\FunctionTok{rm}\NormalTok{(amazon\_orig\_train, amazon\_orig\_test, amazon\_train, amazon\_val)}
\FunctionTok{rm}\NormalTok{(ids\_train, train\_file\_path, test\_file\_path, val\_file\_path)}
\NormalTok{bert\_preprocess\_model }\OtherTok{\textless{}{-}} \FunctionTok{layer\_hub}\NormalTok{(}
  \AttributeTok{handle =} \StringTok{"https://tfhub.dev/tensorflow/bert\_en\_uncased\_preprocess/3"}\NormalTok{, }\AttributeTok{trainable =} \ConstantTok{FALSE}\NormalTok{, }\AttributeTok{name=}\StringTok{\textquotesingle{}preprocessing\textquotesingle{}}
\NormalTok{)}
\NormalTok{bert\_model }\OtherTok{\textless{}{-}} \FunctionTok{layer\_hub}\NormalTok{(}
  \AttributeTok{handle =} \StringTok{"https://tfhub.dev/tensorflow/small\_bert/bert\_en\_uncased\_L{-}4\_H{-}512\_A{-}8/2"}\NormalTok{,}
  \AttributeTok{trainable =} \ConstantTok{TRUE}\NormalTok{, }\AttributeTok{name=}\StringTok{\textquotesingle{}BERT\_encoder\textquotesingle{}}
\NormalTok{)}
\NormalTok{input }\OtherTok{\textless{}{-}} \FunctionTok{layer\_input}\NormalTok{(}\AttributeTok{shape=}\FunctionTok{shape}\NormalTok{(), }\AttributeTok{dtype=}\StringTok{"string"}\NormalTok{, }\AttributeTok{name=}\StringTok{\textquotesingle{}text\textquotesingle{}}\NormalTok{)}

\NormalTok{output }\OtherTok{\textless{}{-}}\NormalTok{ input }\SpecialCharTok{\%\textgreater{}\%}
  \FunctionTok{bert\_preprocess\_model}\NormalTok{() }\SpecialCharTok{\%\textgreater{}\%}
\NormalTok{  bert\_model }\SpecialCharTok{\%$\%}
\NormalTok{  pooled\_output }\SpecialCharTok{\%\textgreater{}\%} 
  \FunctionTok{layer\_dropout}\NormalTok{(}\FloatTok{0.1}\NormalTok{) }\SpecialCharTok{\%\textgreater{}\%}
  \CommentTok{\# layer\_dense(units = 16, activation = "relu") \%\textgreater{}\%}
  \FunctionTok{layer\_dense}\NormalTok{(}\AttributeTok{units =} \DecValTok{1}\NormalTok{, }\AttributeTok{activation =} \StringTok{"sigmoid"}\NormalTok{, }\AttributeTok{name=}\StringTok{\textquotesingle{}classifier\textquotesingle{}}\NormalTok{)}

\CommentTok{\# summary(model)}
\NormalTok{model }\OtherTok{\textless{}{-}} \FunctionTok{keras\_model}\NormalTok{(input, output)}
\NormalTok{epochs }\OtherTok{=} \DecValTok{5}
\NormalTok{steps\_per\_epoch }\OtherTok{\textless{}{-}} \FloatTok{2e6}
\NormalTok{num\_train\_steps }\OtherTok{\textless{}{-}}\NormalTok{ steps\_per\_epoch }\SpecialCharTok{*}\NormalTok{ epochs}
\NormalTok{num\_warmup\_steps }\OtherTok{\textless{}{-}} \FunctionTok{as.integer}\NormalTok{(}\FloatTok{0.1}\SpecialCharTok{*}\NormalTok{num\_train\_steps)}

\NormalTok{init\_lr }\OtherTok{\textless{}{-}} \FloatTok{3e{-}5}
\NormalTok{opt }\OtherTok{\textless{}{-}}\NormalTok{ o\_nlp}\SpecialCharTok{$}\NormalTok{optimization}\SpecialCharTok{$}\FunctionTok{create\_optimizer}\NormalTok{(}\AttributeTok{init\_lr=}\NormalTok{init\_lr,}
                                     \AttributeTok{num\_train\_steps=}\NormalTok{num\_train\_steps,}
                                  \AttributeTok{num\_warmup\_steps=}\NormalTok{num\_warmup\_steps,}
                                          \AttributeTok{optimizer\_type=}\StringTok{\textquotesingle{}adamw\textquotesingle{}}\NormalTok{)}

\NormalTok{model }\SpecialCharTok{\%\textgreater{}\%}
  \FunctionTok{compile}\NormalTok{(}
    \AttributeTok{loss =} \StringTok{"binary\_crossentropy"}\NormalTok{,}
    \AttributeTok{optimizer =}\NormalTok{ opt,}
    \AttributeTok{metrics =} \StringTok{"accuracy"}
\NormalTok{  )}

\FunctionTok{summary}\NormalTok{(model)}
\CommentTok{\# 10000 will take approx 40 mins per epoch on my gpu/mem etc}
\CommentTok{\# 1000 will take approx 4 mins per epoch on my gpu/mem etc}

\NormalTok{tr\_count }\OtherTok{\textless{}{-}} \DecValTok{10000}
\NormalTok{take\_tr }\OtherTok{\textless{}{-}} \FloatTok{0.8} \SpecialCharTok{*}\NormalTok{ tr\_count}
\NormalTok{train\_slice }\OtherTok{\textless{}{-}}\NormalTok{ train\_dataset  }\SpecialCharTok{\%\textgreater{}\%} 
  \FunctionTok{dataset\_shuffle\_and\_repeat}\NormalTok{(}\AttributeTok{buffer\_size =}\NormalTok{ take\_tr }\SpecialCharTok{*}\NormalTok{ batch\_size) }\SpecialCharTok{\%\textgreater{}\%} 
  \FunctionTok{dataset\_take}\NormalTok{(take\_tr)}

\NormalTok{take\_val }\OtherTok{\textless{}{-}} \FloatTok{0.2} \SpecialCharTok{*}\NormalTok{ tr\_count}
\NormalTok{val\_slice }\OtherTok{\textless{}{-}}\NormalTok{ val\_dataset  }\SpecialCharTok{\%\textgreater{}\%} 
  \FunctionTok{dataset\_shuffle\_and\_repeat}\NormalTok{(}\AttributeTok{buffer\_size =}\NormalTok{ take\_val }\SpecialCharTok{*}\NormalTok{ batch\_size) }\SpecialCharTok{\%\textgreater{}\%} 
  \FunctionTok{dataset\_take}\NormalTok{(take\_val)}
\NormalTok{epochs }\OtherTok{\textless{}{-}} \DecValTok{5}
\NormalTok{seed }\OtherTok{=} \DecValTok{42}

\NormalTok{history }\OtherTok{\textless{}{-}}\NormalTok{ model }\SpecialCharTok{\%\textgreater{}\%} 
  \FunctionTok{fit}\NormalTok{(}
\NormalTok{    train\_slice,}
    \AttributeTok{epochs =}\NormalTok{ epochs,}
    \AttributeTok{validation\_data =}\NormalTok{ val\_slice,}
    \AttributeTok{initial\_epoch =} \DecValTok{0}\NormalTok{,}
    \AttributeTok{verbose =} \DecValTok{2}
\NormalTok{  )}
\FunctionTok{plot}\NormalTok{(history)}
\NormalTok{model }\SpecialCharTok{\%\textgreater{}\%} \FunctionTok{evaluate}\NormalTok{(test\_dataset)}
\NormalTok{test\_slice }\OtherTok{\textless{}{-}}\NormalTok{ test\_dataset  }\SpecialCharTok{\%\textgreater{}\%} 
  \FunctionTok{dataset\_take}\NormalTok{(}\DecValTok{100}\NormalTok{)}

\NormalTok{model }\SpecialCharTok{\%\textgreater{}\%} \FunctionTok{evaluate}\NormalTok{(test\_slice)}
\CommentTok{\# Best BERT accuracy}
\NormalTok{bert\_best\_acc }\OtherTok{\textless{}{-}}  \FunctionTok{max}\NormalTok{(history}\SpecialCharTok{$}\NormalTok{metrics}\SpecialCharTok{$}\NormalTok{val\_accuracy) }

\CommentTok{\# Lowest BERT Loss}
\NormalTok{bert\_lowest\_loss }\OtherTok{\textless{}{-}} \FunctionTok{min}\NormalTok{(history}\SpecialCharTok{$}\NormalTok{metrics}\SpecialCharTok{$}\NormalTok{val\_loss)}
\NormalTok{results\_table }\OtherTok{\textless{}{-}} \FunctionTok{bind\_rows}\NormalTok{(results\_table,}
                           \FunctionTok{tibble}\NormalTok{(}\AttributeTok{Index =} \StringTok{"5"}\NormalTok{,}
                                  \AttributeTok{Method =} \StringTok{"BERT"}\NormalTok{,}
                                  \AttributeTok{Accuracy =}\NormalTok{ bert\_best\_acc,}
                                  \AttributeTok{Loss =}\NormalTok{ bert\_lowest\_loss))}

\FunctionTok{kable}\NormalTok{(results\_table, }\StringTok{"simple"}\NormalTok{,}\AttributeTok{caption=}\StringTok{"BERT Model Results}\SpecialCharTok{\textbackslash{}\textbackslash{}}\StringTok{label\{tbl:bert\_results\_table\}"}\NormalTok{)}
\CommentTok{\# \%\textgreater{}\%}
\CommentTok{\#     kable\_styling(latex\_options=c("HOLD\_position"), font\_size=7)}
\CommentTok{\# Stop the clock}
\CommentTok{\# proc.time() {-} ptm}
\FunctionTok{Sys.time}\NormalTok{()}
\NormalTok{    knitr}\SpecialCharTok{::}\FunctionTok{knit\_exit}\NormalTok{()}
\end{Highlighting}
\end{Shaded}

\begin{center}\rule{0.5\linewidth}{0.5pt}\end{center}

\newpage

Terms like generate\index{generate} and some\index{others} will also
show up.

\printindex

\begin{Shaded}
\begin{Highlighting}[]
\NormalTok{    knitr}\SpecialCharTok{::}\FunctionTok{knit\_exit}\NormalTok{()}
\end{Highlighting}
\end{Shaded}


\end{document}