MFTE_Python_Eval_Results.Rmd

---
title: "Python MFTE evaluation"
author: "Elen Le Foll"
date: "06/05/2023"
output: html_document
---

```{r setup, include=FALSE}
knitr::opts_chunk$set(echo = TRUE,
                      message = FALSE,
                      warning = FALSE)

library(DescTools)
library(caret)
library(here)
library(paletteer)
library(readxl)
library(svglite)
library(tidyverse)

# Set the random number generator seed for reproducibility.
set.seed(13)

```

# Data import 

These chunks import the data directly from the Excel files in which we did the manual tag check and corrections. Warning messages that Tag4 columns are unknown or uninitialised can safely be ignored.

```{r import-functions, include=FALSE}

importEval3 <- function(file, fileID, register, corpus) {
  Tag1 <- file %>% 
  add_column(FileID = fileID, Register = register, Corpus = corpus) %>%
  select(FileID, Corpus, Register, Output, Token, Tag1, CorrectedTag1) %>% 
  rename(Tag = Tag1, TagGold = CorrectedTag1, Token = Token) %>% 
  mutate(Evaluation = ifelse(is.na(TagGold), TRUE, FALSE)) %>% 
  mutate(TagGold = ifelse(is.na(TagGold), as.character(Tag), as.character(TagGold))) %>%
  filter(!is.na(Tag)) %>% 
  mutate_if(is.character, as.factor)
  
  Tag2 <- file %>% 
  add_column(FileID = fileID, Register = register, Corpus = corpus) %>%
  select(FileID, Corpus, Register, Output, Token, Tag2, CorrectedTag2) %>% 
  rename(Tag = Tag2, TagGold = CorrectedTag2, Token = Token) %>% 
  mutate(Evaluation = ifelse(is.na(TagGold), TRUE, FALSE)) %>% 
  mutate(TagGold = ifelse(is.na(TagGold), as.character(Tag), as.character(TagGold))) %>%
  mutate(Tag = ifelse(is.na(Tag) & !is.na(TagGold), "NONE", as.character(Tag))) %>%
  filter(!is.na(Tag)) %>% 
  mutate_if(is.character, as.factor)

  Tag3 <- file %>% 
  add_column(FileID = fileID, Register = register, Corpus = corpus) %>%
  select(FileID, Corpus, Register, Output, Token, Tag3, CorrectedTag3) %>% 
  rename(Tag = Tag3, TagGold = CorrectedTag3, Token = Token) %>% 
  mutate(Evaluation = ifelse(is.na(TagGold), TRUE, FALSE)) %>% 
  mutate(TagGold = ifelse(is.na(TagGold), as.character(Tag), as.character(TagGold))) %>%
  mutate(Tag = ifelse(is.na(Tag) & !is.na(TagGold), "NONE", as.character(Tag))) %>%
  filter(!is.na(Tag)) %>% 
  mutate_if(is.character, as.factor)

output <- rbind(Tag1, Tag2, Tag3) %>% 
  mutate(across(where(is.factor), str_remove_all, pattern = fixed(" "))) %>% # Removes all white spaces which are found in the excel files
  filter(!is.na(Output)) %>% 
  mutate_if(is.character, as.factor)

}

importEval4 <- function(file, fileID, register, corpus) {
  Tag1 <- file %>% 
  add_column(FileID = fileID, Register = register, Corpus = corpus) %>%
  select(FileID, Corpus, Register, Output, Token, Tag1, CorrectedTag1) %>% 
  rename(Tag = Tag1, TagGold = CorrectedTag1, Token = Token) %>% 
  mutate(Evaluation = ifelse(is.na(TagGold), TRUE, FALSE)) %>% 
  mutate(TagGold = ifelse(is.na(TagGold), as.character(Tag), as.character(TagGold))) %>%
  filter(!is.na(Tag)) %>% 
  mutate_if(is.character, as.factor)
  
  Tag2 <- file %>% 
  add_column(FileID = fileID, Register = register, Corpus = corpus) %>%
  select(FileID, Corpus, Register, Output, Token, Tag2, CorrectedTag2) %>% 
  rename(Tag = Tag2, TagGold = CorrectedTag2, Token = Token) %>% 
  mutate(Evaluation = ifelse(is.na(TagGold), TRUE, FALSE)) %>% 
  mutate(TagGold = ifelse(is.na(TagGold), as.character(Tag), as.character(TagGold))) %>%
  mutate(Tag = ifelse(is.na(Tag) & !is.na(TagGold), "NONE", as.character(Tag))) %>%
  filter(!is.na(Tag)) %>% 
  mutate_if(is.character, as.factor)

Tag3 <- file %>% 
  add_column(FileID = fileID, Register = register, Corpus = corpus) %>%
  select(FileID, Corpus, Register, Output, Token, Tag3, CorrectedTag3) %>% 
  rename(Tag = Tag3, TagGold = CorrectedTag3, Token = Token) %>% 
  mutate(Evaluation = ifelse(is.na(TagGold), TRUE, FALSE)) %>% 
  mutate(TagGold = ifelse(is.na(TagGold), as.character(Tag), as.character(TagGold))) %>%
  mutate(Tag = ifelse(is.na(Tag) & !is.na(TagGold), "NONE", as.character(Tag))) %>%
  filter(!is.na(Tag)) %>% 
  mutate_if(is.character, as.factor)

Tag4 <- file %>% 
  add_column(FileID = fileID, Register = register, Corpus = corpus) %>%
  select(FileID, Corpus, Register, Output, Token, Tag4, CorrectedTag4) %>% 
  rename(Tag = Tag4, TagGold = CorrectedTag4, Token = Token) %>% 
  mutate(Evaluation = ifelse(is.na(TagGold), TRUE, FALSE)) %>% 
  mutate(TagGold = ifelse(is.na(TagGold), as.character(Tag), as.character(TagGold))) %>%
  mutate(Tag = ifelse(is.na(Tag) & !is.na(TagGold), "NONE", as.character(Tag))) %>%
  filter(!is.na(Tag)) %>% 
  mutate_if(is.character, as.factor)

output <- rbind(Tag1, Tag2, Tag3, Tag4) %>% 
  mutate(across(where(is.factor), str_remove_all, pattern = fixed(" "))) %>% # Removes all white spaces which are found in the excel files
  filter(!is.na(Tag)) %>% 
  mutate_if(is.character, as.factor)

}

importEval <- function(file, fileID, register, corpus) { 
  if(sum(!is.na(file$Tag4)) > 0) {
    output = importEval4(file = file, fileID = fileID, register = register, corpus = corpus)
  }
  else{
    output = importEval3(file = file, fileID = fileID, register = register, corpus = corpus)
  }
}

```

```{r import-individual-files, include=FALSE}

BNC_AcaHumBk34 <- importEval(file = read_excel(here("evaluation", "BNC_AcaHumBk34.xlsx"), col_types = "text"), fileID = "AcaHumBk34", register = "academic", corpus = "BNC2014")

BNC_BAcjH78 <- importEval(file = read_excel(here("evaluation", "BNC_BAcjH78.xlsx"), col_types = "text"), fileID = "BNC_BAcjH78", register = "academic", corpus = "BNC2014")

BNC_BAcjM107 <- importEval(file = read_excel(here("evaluation", "BNC_BAcjM107.xlsx"), col_types = "text"), fileID = "BNC_BAcjM107", register = "academic", corpus = "BNC2014")

BNC_BEBl293 <- importEval(file = read_excel(here("evaluation", "BNC_BEBl293.xlsx"), col_types = "text"), fileID = "BNC_BEBl293", register = "e-language", corpus = "BNC2014")

BNC_BEEm76 <- importEval(file = read_excel(here("evaluation", "BNC_BEEm76.xlsx"), col_types = "text"), fileID = "BNC_BEEm76", register = "e-language", corpus = "BNC2014")

BNC_BERe31 <- importEval(file = read_excel(here("evaluation", "BNC_BERe31.xlsx"), col_types = "text"), fileID = "BNC_BERe31", register = "e-language", corpus = "BNC2014")

BNC_BFict_b2 <- importEval(file = read_excel(here("evaluation", "BNC_BFict_b2.xlsx"), col_types = "text"), fileID = "BNC_BFict_b2", register = "fiction", corpus = "BNC2014")

BNC_BMass311 <- importEval(file = read_excel(here("evaluation", "BNC_BMass311.xlsx"), col_types = "text"), fileID = "BNC_BMass311", register = "news", corpus = "BNC2014")

BNC_BReg495 <- importEval(file = read_excel(here("evaluation", "BNC_BReg495.xlsx"), col_types = "text"), fileID = "BNC_BReg495", register = "news", corpus = "BNC2014")

BNC_BSer145 <- importEval(file = read_excel(here("evaluation", "BNC_BSer145.xlsx"), col_types = "text"), fileID = "BNC_BSer145", register = "news", corpus = "BNC2014")

BNC_ElanBlogBla12 <- importEval(file = read_excel(here("evaluation", "BNC_ElanBlogBla12.xlsx"), col_types = "text"), fileID = "BNC_ElanBlogBla12", register = "e-language", corpus = "BNC2014")

BNC_ElanBlogSlu30 <- importEval(file = read_excel(here("evaluation", "BNC_ElanBlogSlu30.xlsx"), col_types = "text"), fileID = "BNC_ElanBlogSlu30", register = "e-language", corpus = "BNC2014")

BNC_ElanEmail102 <- importEval(file = read_excel(here("evaluation", "BNC_ElanEmail102.xlsx"), col_types = "text"), fileID = "BNC_ElanEmail102", register = "e-language", corpus = "BNC2014")

BNC_ElanForumCar5 <- importEval(file = read_excel(here("evaluation", "BNC_ElanForumCar5.xlsx"), col_types = "text"), fileID = "BNC_ElanForumCar5", register = "e-language", corpus = "BNC2014")

BNC_ElanForumRig1 <- importEval(file = read_excel(here("evaluation", "BNC_ElanForumRig1.xlsx"), col_types = "text"), fileID = "BNC_ElanForumRig1", register = "e-language", corpus = "BNC2014")

BNC_ElanRev27 <- importEval(file = read_excel(here("evaluation", "BNC_ElanRev27.xlsx"), col_types = "text"), fileID = "BNC_ElanRev27", register = "e-language", corpus = "BNC2014")

BNC_ElanSms33 <- importEval(file = read_excel(here("evaluation", "BNC_ElanSms33.xlsx"), col_types = "text"), fileID = "BNC_ElanSms33", register = "e-language", corpus = "BNC2014")

BNC_ElanSocFac4_pt1 <- importEval(file = read_excel(here("evaluation", "BNC_ElanSocFac4_pt1.xlsx"), col_types = "text"), fileID = "BNC_ElanSocFac4_pt1", register = "e-language", corpus = "BNC2014")

BNC_ElanSocTwi6_pt4 <- importEval(file = read_excel(here("evaluation", "BNC_ElanSocTwi6_pt4.xlsx"), col_types = "text"), fileID = "BNC_ElanSocTwi6_pt4", register = "e-language", corpus = "BNC2014")

BNC_ElanSocTwi49_pt7 <- importEval(file = read_excel(here("evaluation", "BNC_ElanSocTwi49_pt7.xlsx"), col_types = "text"), fileID = "BNC_ElanSocTwi49_pt7", register = "e-language", corpus = "BNC2014")

BNC_FictFan41 <- importEval(file = read_excel(here("evaluation", "BNC_FictFan41.xlsx"), col_types = "text"), fileID = "BNC_FictFan41", register = "fiction", corpus = "BNC2014")

BNC_FictMis228 <- importEval(file = read_excel(here("evaluation", "BNC_FictMis228.xlsx"), col_types = "text"), fileID = "BNC_FictMis228", register = "fiction", corpus = "BNC2014")

BNC_MagAut1397 <- importEval(file = read_excel(here("evaluation", "BNC_MagAut1397.xlsx"), col_types = "text"), fileID = "BNC_MagAut1397", register = "news", corpus = "BNC2014")

BNC_MagPc275 <- importEval(file = read_excel(here("evaluation", "BNC_MagPc275.xlsx"), col_types = "text"), fileID = "BNC_MagPc275", register = "news", corpus = "BNC2014")

BNC_NewMaDas2819 <- importEval(file = read_excel(here("evaluation", "BNC_NewMaDas2819.xlsx"), col_types = "text"), fileID = "BNC_NewMaDas2819", register = "news", corpus = "BNC2014")

BNC_NewReBet1393 <- importEval(file = read_excel(here("evaluation", "BNC_NewReBet1393.xlsx"), col_types = "text"), fileID = "BNC_NewReBet1393", register = "news", corpus = "BNC2014")

BNC_NewSeGua553 <- importEval(file = read_excel(here("evaluation", "BNC_NewSeGua553.xlsx"), col_types = "text"), fileID = "BNC_NewSeGua553", register = "news", corpus = "BNC2014")

BNC_Sp2m0f33 <- importEval(file = read_excel(here("evaluation", "BNC_Sp2m0f33.xlsx"), col_types = "text"), fileID = "BNC_Sp2m0f33", register = "spoken", corpus = "BNC2014")

BNC_Sp2m2f63 <- importEval(file = read_excel(here("evaluation", "BNC_Sp2m2f63.xlsx"), col_types = "text"), fileID = "BNC_Sp2m2f63", register = "spoken", corpus = "BNC2014")

BNC_Sp3m1f10 <- importEval(file = read_excel(here("evaluation", "BNC_Sp3m1f10.xlsx"), col_types = "text"), fileID = "BNC_Sp3m1f10", register = "spoken", corpus = "BNC2014")

# Command to import all COCA files from directory with fileID, register and corpus

# get file names from directory
# files <- list.files(here("evaluation"))

# split to save names; name for data frame will be first element
# names <- strsplit(files, "\\.")

# now get the files
# for (i in 1:length(files)) { # for each file in the list
#    fileName <- files[[i]] # save filename of element i
#    dataName <- names[[i]][[1]] # save data name of element i
#    tempData <- importEval(file = read_excel(here("evaluation", "fileName"), col_types = "text"), fileID = dataName, register = "spoken", corpus = "COCA")
#    assign (dataName, tempData, envir=.GlobalEnv)  # assign the results of file to the data named
# 
# }

COCA_acad_4000541 <- importEval(file = read_excel(here("evaluation", "COCA_acad_4000541.xlsx"), col_types = "text"), fileID = "COCA_acad_4000541", register = "academic", corpus = "COCA")

COCA_acad_4017541 <- importEval(file = read_excel(here("evaluation", "COCA_acad_4017541.xlsx"), col_types = "text"), fileID = "COCA_acad_4017541", register = "academic", corpus = "COCA")

COCA_acad_4170341 <- importEval(file = read_excel(here("evaluation", "COCA_acad_4170341.xlsx"), col_types = "text"), fileID = "COCA_acad_4170341", register = "academic", corpus = "COCA")

COCA_blog_5157941 <- importEval(file = read_excel(here("evaluation", "COCA_blog_5157941.xlsx"), col_types = "text"), fileID = "COCA_blog_5157941", register = "e-language", corpus = "COCA")

COCA_blog_5174141 <- importEval(file = read_excel(here("evaluation", "COCA_blog_5174141.xlsx"), col_types = "text"), fileID = "COCA_blog_5174141", register = "e-language", corpus = "COCA")

COCA_blog_5176541 <- importEval(file = read_excel(here("evaluation", "COCA_blog_5176541.xlsx"), col_types = "text"), fileID = "COCA_blog_5176541", register = "e-language", corpus = "COCA")

COCA_fict_1000441 <- importEval(file = read_excel(here("evaluation", "COCA_fict_1000441.xlsx"), col_types = "text"), fileID = "COCA_fict_1000441", register = "fiction", corpus = "COCA")

COCA_fict_1003141 <- importEval(file = read_excel(here("evaluation", "COCA_fict_1003141.xlsx"), col_types = "text"), fileID = "COCA_fict_1003141", register = "fiction", corpus = "COCA")

COCA_fict_5003241 <- importEval(file = read_excel(here("evaluation", "COCA_fict_5003241.xlsx"), col_types = "text"), fileID = "COCA_fict_5003241", register = "fiction", corpus = "COCA")

COCA_mag_2029741 <- importEval(file = read_excel(here("evaluation", "COCA_mag_2029741.xlsx"), col_types = "text"), fileID = "COCA_mag_2029741", register = "news", corpus = "COCA")

COCA_mag_2030941 <- importEval(file = read_excel(here("evaluation", "COCA_mag_2030941.xlsx"), col_types = "text"), fileID = "COCA_mag_2030941", register = "news", corpus = "COCA")

COCA_mag_4180341 <- importEval(file = read_excel(here("evaluation", "COCA_mag_4180341.xlsx"), col_types = "text"), fileID = "COCA_mag_4180341", register = "news", corpus = "COCA")

COCA_News_4087357 <- importEval(file = read_excel(here("evaluation", "COCA_News_4087357.xlsx"), col_types = "text"), fileID = "COCA_News_4087357", register = "news", corpus = "COCA")

COCA_News_4087464 <- importEval(file = read_excel(here("evaluation", "COCA_News_4087464.xlsx"), col_types = "text"), fileID = "COCA_News_4087464", register = "news", corpus = "COCA")

COCA_News_4087649 <- importEval(file = read_excel(here("evaluation", "COCA_News_4087649.xlsx"), col_types = "text"), fileID = "COCA_News_4087649", register = "news", corpus = "COCA")

COCA_News_4087995 <- importEval(file = read_excel(here("evaluation", "COCA_News_4087995.xlsx"), col_types = "text"), fileID = "COCA_News_4087995", register = "news", corpus = "COCA")

COCA_Opinion_4061065 <- importEval(file = read_excel(here("evaluation", "COCA_Opinion_4061065.xlsx"), col_types = "text"), fileID = "COCA_Opinion_4061065", register = "news", corpus = "COCA")

COCA_Opinion_4062489 <- importEval(file = read_excel(here("evaluation", "COCA_Opinion_4062489.xlsx"), col_types = "text"), fileID = "COCA_Opinion_4062489", register = "news", corpus = "COCA")

COCA_Opinion_4079063 <- importEval(file = read_excel(here("evaluation", "COCA_Opinion_4079063.xlsx"), col_types = "text"), fileID = "COCA_Opinion_4079063", register = "news", corpus = "COCA")

COCA_Opinion_4090647 <- importEval(file = read_excel(here("evaluation", "COCA_Opinion_4090647.xlsx"), col_types = "text"), fileID = "COCA_Opinion_4090647", register = "news", corpus = "COCA")

COCA_Spoken_4082518 <- importEval(file = read_excel(here("evaluation", "COCA_Spoken_4082518.xlsx"), col_types = "text"), fileID = "COCA_Spoken_4082518", register = "spoken", corpus = "COCA")

COCA_Spoken_4082551 <- importEval(file = read_excel(here("evaluation", "COCA_Spoken_4082551.xlsx"), col_types = "text"), fileID = "COCA_Spoken_4082551", register = "spoken", corpus = "COCA")

COCA_Spoken_4082571 <- importEval(file = read_excel(here("evaluation", "COCA_Spoken_4082571.xlsx"), col_types = "text"), fileID = "COCA_Spoken_4082571", register = "spoken", corpus = "COCA")

COCA_Spoken_4082646 <- importEval(file = read_excel(here("evaluation", "COCA_Spoken_4082646.xlsx"), col_types = "text"), fileID = "COCA_Spoken_4082646", register = "spoken", corpus = "COCA")

COCA_tvm_5208241 <- importEval(file = read_excel(here("evaluation", "COCA_tvm_5208241.xlsx"), col_types = "text"), fileID = "COCA_tvm_5208241", register = "TV/movies", corpus = "COCA")

COCA_tvm_5215441 <- importEval(file = read_excel(here("evaluation", "COCA_tvm_5215441.xlsx"), col_types = "text"), fileID = "COCA_tvm_5215441", register = "TV/movies", corpus = "COCA")

COCA_tvm_5246241 <- importEval(file = read_excel(here("evaluation", "COCA_tvm_5246241.xlsx"), col_types = "text"), fileID = "COCA_tvm_5246241", register = "TV/movies", corpus = "COCA")

COCA_web_5026941 <- importEval(file = read_excel(here("evaluation", "COCA_web_5026941.xlsx"), col_types = "text"), fileID = "COCA_web_5026941", register = "e-language", corpus = "COCA")

COCA_web_5035341 <- importEval(file = read_excel(here("evaluation", "COCA_web_5035341.xlsx"), col_types = "text"), fileID = "COCA_web_5035341", register = "e-language", corpus = "COCA")

COCA_web_5080941 <- importEval(file = read_excel(here("evaluation", "COCA_web_5080941.xlsx"), col_types = "text"), fileID = "COCA_web_5080941", register = "e-language", corpus = "COCA") 

```

```{r combine-files, include=FALSE}

# Command to rbind all COCA and BNC R objects in the local environment

list_of_dataframes <- objects(pattern = "BNC|COCA")
list_of_dataframes <- toString(objects(pattern = "BNC|COCA"))
list_of_dataframes

EvalData <- rbind(BNC_AcaHumBk34, BNC_BAcjH78, BNC_BAcjM107, BNC_BEBl293, BNC_BEEm76, BNC_BERe31, BNC_BFict_b2, BNC_BMass311, BNC_BReg495, BNC_BSer145, BNC_ElanBlogBla12, BNC_ElanBlogSlu30, BNC_ElanEmail102, BNC_ElanForumCar5, BNC_ElanForumRig1, BNC_ElanRev27, BNC_ElanSms33, BNC_ElanSocFac4_pt1, BNC_ElanSocTwi49_pt7, BNC_ElanSocTwi6_pt4, BNC_FictFan41, BNC_FictMis228, BNC_MagAut1397, BNC_MagPc275, BNC_NewMaDas2819, BNC_NewReBet1393, BNC_NewSeGua553, BNC_Sp2m0f33, BNC_Sp2m2f63, BNC_Sp3m1f10, COCA_acad_4000541, COCA_acad_4017541, COCA_acad_4170341, COCA_blog_5157941, COCA_blog_5174141, COCA_blog_5176541, COCA_fict_1000441, COCA_fict_1003141, COCA_fict_5003241, COCA_mag_2029741, COCA_mag_2030941, COCA_mag_4180341, COCA_News_4087357, COCA_News_4087464, COCA_News_4087649, COCA_News_4087995, COCA_Opinion_4061065, COCA_Opinion_4062489, COCA_Opinion_4079063, COCA_Opinion_4090647, COCA_Spoken_4082518, COCA_Spoken_4082551, COCA_Spoken_4082571, COCA_Spoken_4082646, COCA_tvm_5208241, COCA_tvm_5215441, COCA_tvm_5246241, COCA_web_5026941, COCA_web_5035341, COCA_web_5080941) 

summary(EvalData)
unique(EvalData$FileID)
unique(EvalData$TagGold)
unique(EvalData$Tag)

EvalData <- EvalData %>% 
  mutate(TagGold = ifelse(TagGold == "none", "NONE", as.character(TagGold))) %>%
  mutate(TagGold = as.factor(ifelse(TagGold == "unclear", "UNCLEAR", as.character(TagGold))))

#saveRDS(EvalData, here("evaluation", "MFTE_Python_Eval_Results.rds")) # Last saved 10 August 2023

#write.csv(EvalData, here("evaluation", "MFTE_Python_Eval_Results.csv")) # Last saved 10 August 2023

```

```{r quick-import}

EvalData <- readRDS(here("evaluation", "MFTE_Python_Eval_Results.rds")) # 
summary(EvalData)

# Total number of tags manually checked
nrow(EvalData) # 61154

# Number of tags evaluated per file
EvalData %>% 
  group_by(FileID) %>% 
  count(.) %>% 
  arrange(desc(n))

# Number of tokens evaluated per corpus and register subcorpus
EvalData %>% 
  group_by(Corpus, Register) %>% 
  count() %>% 
  #arrange(-n) %>% 
  as.data.frame()

EvalData %>% 
  group_by(Corpus, Register) %>% 
  count(FileID) %>% 
  print(n = 100)

660+671+761 # BNC blogs
574+1390 # BNC e-mails
1368+1454 # BNC forums
1868+1423 # BNC review
1306+675+1282 # BNC social media
1308+1055 # BNC magazines

745+643+814 # COCA magazines
540+595+606+601 # COCA news articles
556+541+1357+539 # COCA opinion

```

# Analysis

```{r UNCLEARs}

nrow(EvalData) # 61154 tags manually checked
summary(EvalData$TagGold) # 294 UNCLEAR

BinomCI(294, 61154,
        conf.level = 0.95,
        sides = "two.sided",
        method = "wilsoncc") * 100
```

```{r MFTE-features}

data <- EvalData %>% 
  filter(!TagGold %in% c("UNCLEAR","unclear")) %>% 
  filter(!TagGold %in% c("ACT", "NFP", "GW", "HYPH", "ADD", "AFX", "FW", "WQ", "SYM", "WDT", "MDother")) %>% 
  filter(!Tag %in% c("ACT", "NFP", "GW", "HYPH", "ADD", "AFX", "FW", "WQ", "SYM", "WDT", "MDother")) %>% 
  filter(TagGold %in% c(str_extract(Tag, "[A-Z0-9]+"))) %>% # Remove all punctuation tags which are uninteresting here.
  droplevels(.) %>% 
  mutate(Tag = factor(Tag, levels = union(levels(Tag), levels(TagGold)))) %>% # Ensure that the factor levels are the same for the next caret operation
  mutate(TagGold = factor(TagGold, levels = union(levels(Tag), levels(TagGold))))

# Spot gold tag corrections that are not actually errors
data[data$Tag==data$TagGold & data$Evaluation == FALSE,] %>% as.data.frame() # Should be zero!

nrow(EvalData) - nrow(data) # 8506 tags excluded from further analysis because not tallied in MFTE tabular outputs
nrow(data) # Number of relevant tags (actually included in MFTE outputs) = 52648
summary(data$Evaluation) # Of those: 51139 = TRUE

BinomCI(51139, 52648,
        conf.level = 0.95,
        sides = "two.sided",
        method = "wilsoncc") * 100

summary(data$TagGold)
levels(data$TagGold) # Should be 73 features

#saveRDS(data, here("evaluation", "MFTE_Python_Eval_Results_filtered.rds")) # Last saved 10 August 2023

#write.csv(data, here("evaluation", "MFTE_Python_Eval_Results_filtered.csv")) # Last saved 10 August 2023

```

```{r recall-precision-f1}

cm <- caret::confusionMatrix(data$Tag, data$TagGold) # Create confusion matrix
cm$overall 

cm.features <- round((cm$byClass[,5:7])*100, 2)

cm.features %>% 
  as.data.frame %>% 
  arrange(-F1)

cm.features %>% 
  as.data.frame %>% 
  filter(Precision < 90 | Recall < 90) %>% 
  arrange(-F1)

```

# Visualising tagger errors 

```{r, fig.width = 8, fig.height = 8}

min_n <- 100 # This can be dropped to 0 in order to print the full data matrix
jitter_dist <- 0.2
opacity <- 0.3

data_filtered1 <- EvalData %>% 
  filter(!TagGold %in% c("UNCLEAR","unclear")) %>% 
  filter(!TagGold %in% c("ACT", "NFP", "GW", "HYPH", "ADD", "AFX", "FW", "WQ", "SYM")) %>% 
  filter(TagGold %in% c(str_extract(Tag, "[A-Z0-9]+"))) %>% # Remove all punctuation tags which are uninteresting here. 
  add_count(Tag, name = "n_tagged") %>%
  add_count(TagGold, name = "n_tagged_gold") %>%
  filter(
    n_tagged >= min_n,
    n_tagged_gold >= min_n)

tags_remaining <- union(
  unique(data_filtered1$Tag),
  unique(data_filtered1$TagGold)
)

data_filtered2 <- data_filtered1 %>%
  mutate(
    Tag = factor(Tag, levels = tags_remaining),
    TagGold = factor(TagGold, levels = tags_remaining)) %>% 
  arrange(TagGold)

library(extrafont)

error_fig <- data_filtered2 %>%
  ggplot(aes(x = TagGold, y = Tag, colour = Evaluation)) +
  theme_bw() +
  theme(axis.text.x = element_text(angle = 90, vjust = 0.5, hjust=1),
        text=element_text(family = "Times New Roman"),
        legend.position = "none") +
  scale_color_manual(values = c("red2", "chartreuse3")) + 
  coord_fixed() +
  scale_x_discrete(drop = FALSE) +
  scale_y_discrete(drop = FALSE) +
  geom_jitter(
    #aes(size = n_tagged_gold),
    width = jitter_dist,
    height = jitter_dist,
    alpha = opacity)

error_fig

#ggsave(here("plots", "TaggerErrorMatrix_TimesRoman.svg"), width = 9, height = 9)

```

## Comparing tagger accuracy across different registers

```{r register-based-accuracy}

registerEval <- function(data, register) {
  d <- data %>% filter(Register==register)
  cm <- caret::confusionMatrix(d$Tag, d$TagGold) 
  return(round((cm$overall*100), 2))
  #return(cm$byClass[,5:7])
}

summary(data$Register)

registerEval(data, "academic")
registerEval(data, "e-language")
registerEval(data, "fiction")
registerEval(data, "news")
registerEval(data, "spoken")
registerEval(data, "TV/movies")

dataSpoken <- data %>% 
  filter(Register=="spoken" | Register=="TV/movies")
cmSpoken <- caret::confusionMatrix(dataSpoken$Tag, dataSpoken$TagGold) 
round((cmSpoken$overall*100), 2)

```

```{r variety-based}

varietyEval <- function(data, variety) {
  d <- data %>% filter(Corpus==variety)
  cm <- caret::confusionMatrix(d$Tag, d$TagGold) 
  return(round((cm$overall*100), 2))
  #return(cm$byClass[,5:7])
}

varietyEval(data, "BNC2014")
varietyEval(data, "COCA")

```


## Comparing tagger accuracy across individual files

Note that this is not extremely informative because the individual test files really are quite short.

```{r file-based-accuracy}

fileEval <- function(data, file) {
  d <- data %>% filter(FileID==file) %>% 
    # Ensure that the factor levels are the same for the next caret operation
    mutate(Tag = factor(Tag, levels = union(levels(Tag), levels(TagGold)))) %>% 
  mutate(TagGold = factor(TagGold, levels = union(levels(Tag), levels(TagGold))))
  cm <- caret::confusionMatrix(d$Tag, d$TagGold) 
  return(cm$overall)
  #return(cm$byClass[,5:7])
}

levels(data$FileID)

fileEval(data, "COCA_Opinion_4079063")
fileEval(data, "BNC_BAcjH78")

```

## Compute accuracy metrics per feature

The three accuracy metrics are recall, precision and F1 score.

```{r accuracy-per-feature}

cm <- caret::confusionMatrix(data$Tag, data$TagGold) 
cm$overall
cm$byClass[,5:7]

confusion_matrix <- cm$table
total <- sum(confusion_matrix)
number_of_classes <- nrow(confusion_matrix)
correct <- diag(confusion_matrix)
total_actual_class <- apply(confusion_matrix, 2, sum)
total_pred_class <- apply(confusion_matrix, 1, sum)
# Precision = TP / all that were predicted as positive
precision <- correct / total_pred_class
# Recall = TP / all that were actually positive
recall <- correct / total_actual_class
# F1
f1 <- (2 * precision * recall) / (precision + recall)
# create data frame to output results
results <- data.frame(precision, recall, f1)
results

#write.csv(results, here("evaluation", "MFTEAccuracyResults.csv"))

```

## Compute accuracy metrics with bootstrapping

The idea of computing 95% confidence intervals on these accuracy metrics was inspired by the method and code presented in Picoral et al. (2021). Bootstrapping is applied to every combination of feature and metric to obtain 95% confidence intervals. The code to do so was originally written in R; however because it was incredibly slow and the {boot} library appeared to have some weird bugs that caused various errors, it was translated to run in Python. The script is included in the project's repository and is entitled: `bootstrap_eval.py`.

The results of the `bootstrap_eval.py` Python script are plotted in the following chunk:

```{r plot-accuracy-CI, fig.width = 8, fig.height = 12}
# Import 95 CI data computed in `bootstrap_eval.py`.
resultsCI <- read.csv(here("evaluation", "MFTE_Python_Eval_CIs.csv"), 
                      stringsAsFactors = TRUE)
head(resultsCI) # Check import
str(resultsCI) # Check factors

# Remove NONE tag which was needed to calculate recall and precision but is not, in of itself of relevance to the evaluation results
resultsCI <- resultsCI %>% 
  filter(tag != "NONE") %>% 
  filter(tag != "VB") %>% # This feature is not listed in table of features and should therefore have been excluded earlier on
  filter(tag != "ACT") %>%  # This feature was moved to the extended tagset along with all other semantic verb tags and was therefore not included in the formal evaluation 
  droplevels()

# Plot the accuracy metrics with bootstrapped 95% confidence intervals (portrait-format plot) 
ggplot(resultsCI, aes(y = reorder(tag, desc(tag)), 
                      x = value, 
                      group = metric, 
                      colour = n)) +
  geom_point() +
  geom_errorbar(aes(xmin=lower, xmax = upper)) +
  ylab("") +
  xlab("") +
  facet_wrap(~ factor(metric, c("precision", "recall", "f1"))) +
  scale_color_paletteer_c("harrypotter::harrypotter", 
                          trans = "log", 
                          breaks = c(110, 500, 2000, 10000), 
                          labels = c("100", "500", "2,000", "10,000"), 
                          name = "Number of tokens manually evaluated\n") +
  theme_bw() +
  theme(legend.position = "bottom",
        legend.key.width = unit(1, 'cm'), 
        text = element_text(family = "Times New Roman",
                            size = 15))

#ggsave(here("plots", "TaggerAccuracyResults95CI_portrait_TimesRoman.svg"), width = 8, height = 12) # For publication
```


```{r eval = FALSE, echo = FALSE}
# Plot the accuracy metrics with bootstrapped 95% confidence intervals (landscape-format plot)

# In order to have the tag labels on both the left and right-hand side of the plot, it is necessary to plot the tags as a numeric variable because the ggplot2 function for scale_x_discrete does not have a sec_axis option. This workaround was found here: https://stackoverflow.com/questions/45361904/duplicating-and-modifying-discrete-axis-in-ggplot2
tags1 <- levels(resultsCI$tag)

ggplot(resultsCI, aes(y = as.numeric(reorder(tag, desc(tag))), 
                      x = value, 
                      group = metric, 
                      colour = n)) +
  geom_point() +
  scale_y_continuous(breaks = 1:length(tags1),
                     labels = rev(tags1),
                     expand = c(0.001,0.001),
                     sec.axis = sec_axis(~.,
                                         breaks = 1:length(tags1),
                                         labels = rev(tags1))) +
  geom_errorbar(aes(xmin=lower, xmax = upper)) +
  ylab("") +
  xlab("") +
  facet_wrap(~ factor(metric, c("precision", "recall", "f1"))) +
  scale_color_paletteer_c("harrypotter::harrypotter", 
                          trans = "log", 
                          breaks = c(110, 500, 2000, 10000), 
                          labels = c("100", "500", "2,000", "10,000"), 
                          name = "Number of tokens\nmanually evaluated") +
  theme_bw() +
  theme(legend.position = "right")

#ggsave(here("plots", "TaggerAccuracyResults95CI_landscape.svg"), width = 12, height = 8) # Landscape format, better for presentation slides
```

## Qualitative analysis of tagger errors

```{r errors}

# Adding an error tag with the incorrectly assigned tag and underscore and then the correct "gold" label
errors <- data %>% 
  filter(Evaluation=="FALSE") %>% 
  mutate(Error = paste(Tag, TagGold, sep = " -> ")) 

# Total number of errors
nrow(errors) # 1509

FreqErrors <- errors %>% 
  count(Error) %>% 
  arrange(desc(n)) 

FreqErrors %>% 
  #group_by(Register) %>% 
  filter(n > 9) %>% 
  print.data.frame() 

errors %>% 
  filter(Error == "NN -> JJAT") %>% 
  select(-Output, -Corpus, -Tag, -TagGold) %>% 
  filter(grepl(x = Token, pattern = "[A-Z]+.")) %>% 
  print.data.frame() 

errors %>% 
  filter(Error %in% c("NN -> VB", "VB -> NN", "NN -> VPRT", "VPRT -> NN")) %>% 
  count(Token) %>% 
  arrange(desc(n)) %>% 
  print.data.frame() 

errors %>% 
  filter(Error == "NN -> JJPR") %>% 
  count(Token) %>% 
  filter(grepl(x = Token, pattern = "[A-Z]+.")) %>% 
  arrange(desc(n)) %>% 
  print.data.frame() 

```