Scraper.Rmd

---
title: "Scraper"
output: html_notebook
---

```{r setup}
library(tidyverse)
library(rvest)
library(stringr)
library(readxl)

# Helper function to clean the legislator's name
# NO LONGER NEEDED
name_cleaner <- function(name){
  name <- gsub("U.S. Representative", "", name)
  name <- gsub("Representative", "", name)
  name <- gsub("U.S. Congressman", "", name)
  name <- gsub("U.S. Congresswoman", "", name)
  name <- gsub("Congressman", "", name)
  name <- gsub("Congresswoman", "", name)
  name <- gsub("Home - ", "", name)
  name <- gsub("\\s*,?\\s*(D\\.D\\.S\\.|PhD|M\\.D\\.|M\\.Sc\\.|LL\\.M\\.|
               J\\.D\\.|B\\.Sc\\.|D\\.Phil\\.|Ed\\.D\\.|M\\.Eng\\.|\\w{1,3}
               \\.\\w{1,3}\\.)", "", name) # Remove degrees but keep things like Jr.
  name <- str_trim(name)
  
  return(name)
}

# Helper function to detect date formats a la ChatGPT
detect_date_format <- function(date_strings) {
  # A list of candidate formats (date-only and datetime variants)
  candidate_formats <- c(
    "%Y-%m-%dT%H:%M:%S",   # ISO 8601 with T separator
    "%Y-%m-%d %H:%M:%S",   # Common datetime with space separator
    "%Y-%m-%d",            # ISO date
    "%d/%m/%Y %H:%M:%S",   # Day/Month/Year with datetime
    "%d/%m/%Y",            # Day/Month/Year
    "%m/%d/%Y %H:%M:%S",   # Month/Day/Year with datetime
    "%m/%d/%Y",            # Month/Day/Year
    "%Y/%m/%d %H:%M:%S",   # Year/Month/Day with datetime using slash
    "%Y/%m/%d",            # Year/Month/Day with slash
    
    # Full month name formats like "September 23, 2020"
    "%B %d, %Y",
    "%B %d, %Y %H:%M:%S",

    # Abbreviated month name formats like "Sep 23, 2021" (without a period)
    "%b %d, %Y",
    "%b %d, %Y %H:%M:%S",
    
    # Abbreviated month name with a period: "Sep. 23, 2021"
    "%b. %d, %Y",
    "%b. %d, %Y %H:%M:%S"
  )
  
  # For each candidate, count how many strings can be parsed (ignoring warnings)
  valid_counts <- sapply(candidate_formats, function(fmt) {
    parsed <- suppressWarnings(strptime(date_strings, format = fmt))
    sum(!is.na(parsed))
  })
  
  # If none of the formats yield any valid dates, return NA
  if (all(valid_counts == 0)) {
    return(NA)
  }
  
  # Otherwise, choose the candidate format with the most valid parses.
  # In case of ties, which.max() returns the first maximum.
  best_index <- which.max(valid_counts)
  candidate_formats[best_index]
}


```

```{r}
failed_URLs2
```


# Scraper

```{r}
scraper(link="https://nunn.house.gov",
         # start_date="2023-07-20",
         # end_date="2023-10-21")
         start_date="2024-12-01",
         end_date="2024-12-31", debug=T)

```

```{r}
# Helper function that recursively scrapes press releases across pages within
# the given date range
page_scraper <- function(page_index, link, home, start_date, end_date,
                         template, debug=F, page_limit=1000){
  
  if(page_index >= page_limit) stop(paste0("Pagination limit (", page_limit, ") reached for ", home))
  
  # Give the user a message to monitor progress
  # (but don't overwrite previous message in debug mode)
  message(paste0(ifelse(debug, "", "\014"),"Scraping pressers from: ", 
                            home, "    ", "pg: ", page_index))
  
  # Add one day onto the start and end dates to ensure we don't miss any
  # pressers if there are multiple in one day and one happens to be at top or
  # bottom of the page
  start_date <- as.character(as.Date(start_date) - 1)
  end_date <- as.character(as.Date(end_date) + 1)
  
  # Data frame that will hold scraped pressers
  pressers <- data.frame(date=c(), title=c(), body=c(), url=c())
  
  if(template=="A"){
    
    page_link <- paste0(link, "?page=", page_index)
    
    page <- read_html(page_link)
    
    # Get the dates from the press releases previewed on the page
    dates <- page %>% html_nodes(".col-auto:nth-child(1)") %>% 
        html_text() %>% str_trim()
    
    type2 <- F
    if(length(dates)==0){
      type2 <- T
      page_link <- paste0(link, "/page/", page_index, "/")
      page <- read_html(page_link)
      dates <- page %>% html_nodes(".published") %>% html_text() %>% 
        str_trim()
      dates <- sapply(dates, function(x) as.character(strptime(x, f <- "%b %d, %Y", tz = "")))
    }
    
    # Handle empty page case, i.e. when all the pages have been checked
    if(length(dates)==0) return(pressers)
    
    # Format the dates
    if(!type2) dates <- sapply(dates, function(x) as.character(strptime(x, f <- "%B %d, %Y", tz = "")))
  
    if(head(dates, 1) > end_date & tail(dates, 1) > end_date){
      # Haven't reached a page in our date range; recurse on the next page
      return(rbind(pressers, page_scraper(page_index + 1, link, home, 
                                          start_date, end_date, template,  
                                          debug=debug, page_limit=page_limit)))
    }
    else if(head(dates, 1) < start_date){
      # Base case: stop paginating
      return(pressers)
    }
    else{
      # Page has pressers in our date range--scrape them then recurse on next page
      
      # Get indexes for the dates within our range
      desired_dates <- which(dates >= start_date & dates <= end_date)
      
      # If there are pressers from the desired date range on the page then
      # get all the data we want
      if(length(desired_dates)>0){
        
        if(!type2){
          titles <- page %>% html_nodes(".font-weight-bold a") %>% html_text()
          pr_links <- page %>% html_nodes(".font-weight-bold a") %>% 
            html_attr("href") 
          
          # Check if title scraping succeeded, if it didn't then
          # try another method
          if(length(titles)==0){
            titles <- page %>% html_nodes("div.evo-card-title") %>% 
              html_nodes("span") %>% html_text()
          }
          else{
            pr_links <- paste0(home, pr_links)
          }
          
          # Check if link scraping succeeded, if it didn't then
          # try another method
          if(length(pr_links)==0){
            pr_links <- page %>% html_nodes("a") %>% 
            html_attr("href") %>% paste0(home, .)
          }
          
          # Deal with the case where scraping titles and links yields extra junk
          # we don't want (i.e. with simon.house.gov)
          if(length(pr_links)!=length(dates)){
            selection <- which(grepl("/media/press-releases/.+", pr_links))
            pr_links <- pr_links[selection]
            if(length(titles)!=length(dates)) titles <- titles[selection]
          }
 
        }
        else{
          titles <- page %>% html_nodes(".entry-title a") %>% html_text()
          pr_links <- page %>% html_nodes(".entry-title a") %>% 
            html_attr("href")
        }
        
        # Subset to only pressers within date range
        titles <- titles[desired_dates]
        pr_links <- pr_links[desired_dates]
        
        if(!type2){
          # Get the presser text
          texts <- sapply(pr_links, 
                          FUN=function(l){
                            read_html(l) %>% 
                              html_nodes(".evo-press-release__body") %>% 
                              html_text() %>% str_trim
                            })
        }
        else{
          
          # Get the presser text
          texts <- sapply(pr_links, 
                          FUN=function(l){
                            read_html(l) %>% 
                              html_nodes(".et-last-child p") %>% 
                              html_text() %>% 
                              paste(., collapse=" ") %>% 
                              str_trim()
                            })
          
        }
        
        # Add everything to the data frame
        pressers <- rbind(pressers, 
                          data.frame(date=dates[desired_dates], 
                                     title=titles, 
                                     body=texts, url=pr_links))
      }
      
      # Recurse on next page
      return(rbind(pressers, page_scraper(page_index + 1, link, home, 
                                          start_date, end_date, template, 
                                          debug=debug, page_limit=page_limit)))
    }
  }
  else if(template=="B"){
    page_link <- paste0(link, "&Page=", page_index)
    
    page <- read_html(page_link)
    
    # Get the dates from the press releases previewed on the page
    dates <- page %>% html_nodes("time") %>% html_text()
    
    # Handle empty page case, i.e. when all the pages have been checked
    if(length(dates)==0) return(pressers)
    
    # Format the dates
    dates <- sapply(dates, function(x) as.character(strptime(x, f <- "%B %d, %Y", tz = "")))
  
    if(head(dates, 1) > end_date & tail(dates, 1) > end_date){
      # Haven't reached a page in our date range; recurse on the next page
      return(rbind(pressers, page_scraper(page_index + 1, link, home, 
                                          start_date, end_date, template, 
                                          debug=debug, page_limit=page_limit)))
    }
    else if(head(dates, 1) < start_date){
      # Base case: stop paginating
      return(pressers)
    }
    else{
      # Page has pressers in our date range--scrape them then recurse on next page
      
      # Get indexes for the dates within our range
      desired_dates <- which(dates >= start_date & dates <= end_date)
      
      # If there are pressers from the desired date range on the page then
      # get all the data we want
      if(length(desired_dates)>0){
      
        titles <- page %>% html_nodes(".newsie-titler a") %>% html_text()
        pr_links <- page %>% html_nodes(".newsie-titler a") %>% html_attr("href") %>% 
          paste0(home,"/news/", .)
    
        # Subset to only pressers within date range
        titles <- titles[desired_dates]
        pr_links <- pr_links[desired_dates]
        
        # Get the presser text
        texts <- sapply(pr_links, 
                        FUN=function(l){
                          read_html(l) %>% 
                            html_nodes(".bodycopy") %>% 
                            html_text() %>% str_trim
                          })
        
        # Add everything to the data frame
        pressers <- rbind(pressers, 
                          data.frame(date=dates[desired_dates], 
                                     title=titles, 
                                     body=texts, url=pr_links))
      }
      
      # Recurse on next page
      return(rbind(pressers, page_scraper(page_index + 1, link, home, 
                                          start_date, end_date, template, 
                                          debug=debug, page_limit=page_limit)))
    }
  }
  else if(template %in% c("C", "H", "I")){
    page_link <- paste0(link, "/page/", page_index, "/")
    
    page <- read_html(page_link)
    
    # Get the dates from the press releases previewed on the page
    dates <- page %>% html_nodes(".info-date") %>% html_text()
    
    # There are three subtypes of the C and F template, this differentiates them
    type2 <- F
    type3 <- F
    if(identical(character(0), dates) | length(dates)==0){
      dates <- page %>% html_nodes(".date") %>% html_text()
      if(length(dates)!=0) type2 <- T
      else{
        type3 <- T
        dates <- page %>% html_nodes(".time") %>% html_text()
      }
    }

    # Handle empty page case, i.e. when all the pages have been checked
    if(length(dates)==0) return(pressers)
    
    # Extract the date in case there is other junk attached
    dates <- sapply(dates, function(x){
      regmatches(x, gregexpr("[A-Za-z]+(?:\\.|) [0-9]{1,2}, [0-9]{4}", x)) %>% 
        unlist()
                 })
    
    # Format the dates
    if(type3){
      dates <- sapply(dates, 
                    function(x) as.character(
                      strptime(
                        gsub("\\b(\\d+)(st|nd|rd|th)\\b", "\\1", x), 
                        f <- "%b %d, %Y", tz = "")))
    }
    else{
      dates <- sapply(dates, 
                      function(x) as.character(
                        strptime(
                          gsub("\\b(\\d+)(st|nd|rd|th)\\b", "\\1", x), 
                          f <- "%B %d, %Y", tz = "")))
    }
    
    
    if(head(dates, 1) > end_date & tail(dates, 1) > end_date){
      # Haven't reached a page in our date range; recurse on the next page
      return(rbind(pressers, page_scraper(page_index + 1, link, home, 
                                          start_date, end_date, template, 
                                          debug=debug, page_limit=page_limit)))
    }
    else if(head(dates, 1) < start_date){
      # Base case: stop paginating
      return(pressers)
    }
    else{
      # Page has pressers in our date range--scrape them then recurse on next page
      
      # Get indexes for the dates within our range
      desired_dates <- which(dates >= start_date & dates <= end_date)
      
      # If there are pressers from the desired date range on the page then
      # get all the data we want
      if(length(desired_dates)>0){
        
        # Deal with each subtype of the C template
        if(!type2 & !type3){
          titles <- page %>% html_nodes(".info-title") %>% html_text()
          pr_links <- page %>% html_nodes(".btn") %>% html_attr("href")
      
          # Subset to only pressers within date range
          titles <- titles[desired_dates]
          pr_links <- pr_links[desired_dates]
          
          # Get the presser text
          texts <- sapply(pr_links, 
                          FUN=function(l){
                            read_html(l) %>% html_nodes(".news-info__content") %>% 
                              html_text() %>% str_trim
                            })
        }
        else if(type2){
          titles <- page %>% html_nodes(".title") %>% html_text()
          
          # Try a different format
          if(length(titles)==0 | length(titles)!=length(dates)){
            titles <- page %>% html_nodes("h2 a") %>% html_text() %>% str_trim()
            # Try a different different format
            if(length(titles)==0| length(titles)!=length(dates)){
              titles <- page %>% html_nodes("h3") %>% html_text() %>% str_trim()
            }
          }
          
          pr_links <- page %>% html_nodes("a") %>% html_attr("href")
          
          # Try a different format
          if(length(pr_links)==0 | length(pr_links) != length(dates)){
            pr_links <- page %>% html_nodes("h2 a") %>% html_attr("href")
            # Try a different different format
            if(length(pr_links)==0 | length(pr_links) != length(dates)){
              pr_links <- page %>% html_nodes(".btn") %>% html_attr("href")
            }
          }
          
          # Get just the press release links, i.e. house.gov followed by press-releases
          # or a date like 2024/01/27
          pr_links <- grep("house\\.gov(/press-releases/.*|/[0-9]{4}/[0-9]{1,2}/[0-9]{1,2}/.*)", pr_links, value = TRUE)
          
          
          
          # Subset to only pressers within date range
          titles <- titles[desired_dates]
          pr_links <- pr_links[desired_dates]
          
          # Get the presser text
          if(identical(character(0), 
                       read_html(pr_links[1]) %>% 
                       html_nodes("#page") %>% html_text())){
            
            texts <- sapply(pr_links, 
                  FUN=function(l){
                    read_html(l) %>% html_nodes(".content p") %>% 
                      html_text() %>% paste(., collapse=" ") %>% str_trim
                    })
          }
          else{
            texts <- sapply(pr_links, 
                  FUN=function(l){
                    read_html(l) %>% html_nodes("#page") %>% 
                      html_text() %>% str_trim
                    })
          }
        }
        else if(type3){
          titles <- page %>% html_nodes("h2") %>% html_text() %>% str_trim()
          
          pr_links <- page %>% html_nodes(".read-more") %>% html_attr("href")
          
          # Subset to only pressers within date range
          titles <- titles[desired_dates]
          pr_links <- pr_links[desired_dates]
          
          texts <- sapply(pr_links, 
                FUN=function(l){
                  read_html(l) %>% html_nodes(".page-content") %>% 
                    html_text() %>% str_trim()
                  })
        }
        
        # Add everything to the data frame
        pressers <- rbind(pressers, 
                          data.frame(date=dates[desired_dates], 
                                     title=titles, 
                                     body=texts, url=pr_links))
      }
      
      # Recurse on next page
      return(rbind(pressers, page_scraper(page_index + 1, link, home, 
                                          start_date, end_date, template, 
                                          debug=debug, page_limit=page_limit)))
    }
  }
  else if(template=="D"){
    
    page_link <- paste0(link, "?page=", page_index)
    
    page <- read_html(page_link)
    
    # Get the dates from the press releases previewed on the page
    dates <- page %>% html_nodes(".col-auto:nth-child(1)") %>% 
      html_text() %>% str_trim()
    
    # Handle empty page case, i.e. when all the pages have been checked
    if(length(dates)==0) return(pressers)
    
    # Format the dates
    dates <- sapply(dates, 
                    function(x) as.character(
                      strptime(x, f <- "%B %d, %Y", tz = "")))
  
    if(head(dates, 1) > end_date & tail(dates, 1) > end_date){
      # Haven't reached a page in our date range; recurse on the next page
      return(rbind(pressers, page_scraper(page_index + 1, link, home, 
                                          start_date, end_date, template, 
                                          debug=debug, page_limit=page_limit)))
    }
    else if(head(dates, 1) < start_date){
      # Base case: stop paginating
      return(pressers)
    }
    else{
      # Page has pressers in our date range--scrape them then recurse on next page
      
      # Get indexes for the dates within our range
      desired_dates <- which(dates >= start_date & dates <= end_date)
      
      # If there are pressers from the desired date range on the page then
      # get all the data we want
      if(length(desired_dates)>0){
      
        pr_links <- page %>% html_nodes("a") %>% html_attr("href")
        
        # Get rid of unwanted links
        pr_links <- pr_links[grepl("^/media-center/press-releases/.+", pr_links)] %>% 
          paste(home, ., sep="") %>% unique()
        
        
        
        # Subset to only pressers within date range
        pr_links <- pr_links[desired_dates]
        
        # Get the presser titles (difficulty getting it from the landing page)
        titles <- sapply(pr_links, 
                        FUN=function(l){
                          read_html(l) %>% 
                            html_nodes("h1") %>% html_text() %>% str_trim
                          })
        
        # Get the presser text
        texts <- sapply(pr_links, 
                        FUN=function(l){
                          read_html(l) %>% 
                            html_nodes(".evo-press-release__body") %>% 
                            html_text() %>% str_trim
                          })
        
        # Add everything to the data frame
        pressers <- rbind(pressers, 
                          data.frame( 
                                     date=dates[desired_dates], 
                                     title=titles, 
                                     body=texts, url=pr_links))
      }
      
      # Recurse on next page
      return(rbind(pressers, page_scraper(page_index + 1, link, home, 
                                          start_date, end_date, template, 
                                          debug=debug, page_limit=page_limit)))
    }
  }
  else if(template=="E"){
    
    # Template E doesn't play nice with page index of 0
    if(page_index==0) page_index <- 1
    
    page_link <- paste0(link, "?page=", page_index)
    
    page <- read_html(page_link)
    
    # Get the dates from the press releases previewed on the page
    dates <- page %>% html_nodes(".recordListDate") %>% 
      html_text() %>% str_trim()
    
    # Handle empty page case, i.e. when all the pages have been checked
    if(length(dates)==0) return(pressers)
    
    # Format the dates
    dates <- sapply(dates, 
                    function(x) as.character(
                      strptime(x, f <- "%m/%d/%y", tz = "")))
  
    if(head(dates, 1) > end_date & tail(dates, 1) > end_date){
      # Haven't reached a page in our date range; recurse on the next page
      return(rbind(pressers, page_scraper(page_index + 1, link, home, 
                                          start_date, end_date, template, 
                                          debug=debug, page_limit=page_limit)))
    }
    else if(head(dates, 1) < start_date){
      # Base case: stop paginating
      return(pressers)
    }
    else{
      # Page has pressers in our date range--scrape them then recurse on next page
      
      # Get indexes for the dates within our range
      desired_dates <- which(dates >= start_date & dates <= end_date)
      
      # If there are pressers from the desired date range on the page then
      # get all the data we want
      if(length(desired_dates)>0){
      
        pr_links <- page %>% html_nodes(".ContentGrid") %>% html_attr("href") %>% 
          paste0(home,.)
        
        # Subset to only pressers within date range
        pr_links <- pr_links[desired_dates]
        
        # Get the presser titles
        titles <- page %>% html_nodes(".ContentGrid") %>% html_text()
        
        text_node <- ".post-content"
        if(identical(character(0), 
                     read_html(pr_links[1]) %>% 
                            html_nodes(".post-content") %>% 
                            html_text() %>% str_trim)){
          text_node <- ".content"
        }
        
        # Get the presser text
        texts <- sapply(pr_links, 
                        FUN=function(l){
                          read_html(l) %>% 
                            html_nodes(text_node) %>% 
                            html_text() %>% str_trim
                          })
        
        # Add everything to the data frame
        pressers <- rbind(pressers, 
                          data.frame(date=dates[desired_dates], 
                                     title=titles[desired_dates], 
                                     body=texts, url=pr_links))
      }
      
      # Recurse on next page
      return(rbind(pressers, page_scraper(page_index + 1, link, home, 
                                          start_date, end_date, template, 
                                          debug=debug, page_limit=page_limit)))
    }
  }
  # Note: templates F, G, and J are the same, they just have different URL paths to
  # get to the press releases page
  else if(template %in% c("F", "G", "J")){
    
    page_link <- paste0(link, "?page=", page_index)
    
    page <- read_html(page_link)
    
    # Get the dates from the press releases previewed on the page
    dates <- page %>% 
      html_nodes(xpath="//*[contains(concat( \" \", @class, \" \" ), 
                 concat( \" \", \"col-auto\", \" \" ))]") %>% 
      html_text() %>% str_trim()
    
    # Determine if page is a variation of the standard template
    type2 <- F
    if(identical(character(0), dates)){
      
      if(page_index==0) page_index <- 1
      
      page <- read_html(paste0(link, "?PageNum_rs=", page_index))
      
      dates <- page %>% html_nodes(".black") %>% html_text() %>% str_trim()
      
      type2 <- T
    }
    
    # Remove non-date elements
    dates <- dates[which(grepl("^[A-Za-z]+ [0-9]{1,2}, [0-9]{4}$", dates))]
    
    # Handle empty page case, i.e. when all the pages have been checked
    if(length(dates)==0) return(pressers)
    
    # Format the dates
    dates <- sapply(dates, 
                    function(x) as.character(
                      strptime(x, f <- "%B %d, %Y", tz = "")))
  
    if(head(dates, 1) > end_date & tail(dates, 1) > end_date){
      # Haven't reached a page in our date range; recurse on the next page
      return(rbind(pressers, page_scraper(page_index + 1, link, home, 
                                          start_date, end_date, template, 
                                          debug=debug, page_limit=page_limit)))
    }
    else if(head(dates, 1) < start_date){
      # Base case: stop paginating
      return(pressers)
    }
    else{
      # Page has pressers in our date range--scrape them then recurse on next page
      
      # Get indexes for the dates within our range
      desired_dates <- which(dates >= start_date & dates <= end_date)
      
      # If there are pressers from the desired date range on the page then
      # get all the data we want
      if(length(desired_dates)>0){
        
        if(!type2){
          pr_links <- page %>% html_nodes(".font-weight-bold a") %>% 
            html_attr("href") %>% 
            paste0(home,.)
          
          # Subset to only pressers within date range
          pr_links <- pr_links[desired_dates]
          
          # Get the presser titles
          titles <- page %>% html_nodes(".font-weight-bold a") %>% 
            html_text() %>% str_trim()
          
          # Get the presser text
          texts <- sapply(pr_links, 
                          FUN=function(l){
                            read_html(l) %>% 
                              html_nodes(".evo-press-release__body") %>% 
                              html_text() %>% str_trim()
                            })
        }
        else{
          pr_links <- page %>% html_nodes(".title a") %>% 
            html_attr("href") %>% 
            paste0(home,.)
          
          # Subset to only pressers within date range
          pr_links <- pr_links[desired_dates]
          
          # Get the presser titles
          titles <- page %>% html_nodes(".title a") %>% 
            html_text() %>% str_trim()
          
          # Get the presser text
          texts <- sapply(pr_links, 
                          FUN=function(l){
                            read_html(l) %>% 
                              html_nodes("#main_column p") %>% html_text() %>% 
                              paste(., collapse=" ") %>% str_trim()
                            })
        }
        
        # Add everything to the data frame
        pressers <- rbind(pressers, 
                          data.frame(date=dates[desired_dates], 
                                     title=titles[desired_dates], 
                                     body=texts, url=pr_links))
      }
      
      # Recurse on next page
      return(rbind(pressers, page_scraper(page_index + 1, link, home, 
                                          start_date, end_date, template, 
                                          debug=debug, page_limit=page_limit)))
    }
  }
}

# Main scraping function
scraper <- function(link, start_date, end_date, name=NA, page_limit=1000, debug=F){
  
  # Check if link has https:// at the start, if not add it
  if(substr(link, 1, 8)!="https://") link <- paste0("https://", link)
  
  # Check if link is in valid format and date range is valid
  if(!grepl("https://[a-zA-Z0-9\\-]+\\.house\\.gov", link)){
    stop("Link is not of the form \"name.house.gov\" or \"https://name.house.gov\"")
  }
  
  if(start_date > end_date){
    stop("Start date is after end date")
  }
  
  # Sanitize input by removing extra forward slash at end of URL if present
  if(substr(link, nchar(link), nchar(link))=="/") link <- substr(link, 1, nchar(link)-1)
  
  # Data frame that will hold scraped pressers
  pressers <- data.frame(date=c(), title=c(), body=c(), url=c())
  
  # Determine template webpage uses
  template <- NA
  
  page_A <- try(read_html(paste0(link, "/media/press-releases")), 
                silent=T) # Try A
  page_B <- try(read_html(paste0(link, "/news/documentquery.aspx?DocumentTypeID=27")), 
                silent=T) # Try B
  page_C <- try(read_html(paste0(link, "/category/congress_press_release/")), 
                silent=T) # Try C
  page_D <- try(read_html(paste0(link, "/media-center/press-releases")), 
                silent=T) # Try D
  page_E <- try(read_html(paste0(link, "/press-releases")), 
                silent=T) # Try E
  page_F <- try(read_html(paste0(link, "/newsroom/press-releases")),
              silent=T) # Try F
  page_G <- try(read_html(paste0(link, "/news/press-releases")),
            silent=T) # Try G
  page_H <- try(read_html(paste0(link, "/category/press_release/")),
            silent=T) # Try H
  page_I <- try(read_html(paste0(link, "/category/press-releases/")),
            silent=T) # Try I
  page_J <- try(read_html(paste0(link, "/press")), 
            silent=T) # Try J
  
  page_lengths <- c(length(page_A), length(page_B), length(page_C), 
                    length(page_D), length(page_E), length(page_F),
                    length(page_G), length(page_H), length(page_I),
                    length(page_J))
  
  if(debug) print(page_lengths)
  
  # Create a matrix with known lengths of each page template when the other
  # templates' landings are applied
  # When the call to read_html fails (403 or 404 error) it returns a length 1 error object
  # but otherwise the webpage is length 2
  # Rows are labelled with the page template type that results in that pattern
  # Columns are labelled with the tried page template
  length_matrix <- matrix(nrow=22, ncol=10,
                        # Tried  A  B  C  D  E  F  G  H  I  J    Actual
                          data=c(2, 1, 1, 1, 1, 1, 1, 1, 1, 1, # A
                                 2, 1, 1, 1, 1, 1, 1, 1, 1, 2, # A
                                 2, 1, 1, 1, 2, 1, 1, 1, 1, 1, # A
                                 2, 1, 1, 2, 1, 1, 1, 1, 1, 1, # A
                                 2, 1, 1, 2, 2, 2, 2, 1, 2, 1, # A
                                 2, 1, 1, 2, 2, 2, 2, 1, 2, 2, # A 
                                 1, 2, 1, 1, 1, 1, 1, 1, 1, 1, # B
                                 1, 2, 1, 2, 1, 1, 1, 1, 1, 1, # B
                                 2, 2, 1, 1, 1, 1, 1, 1, 1, 1, # B
                                 2, 1, 2, 2, 2, 2, 2, 1, 1, 1, # C
                                 2, 1, 2, 2, 2, 1, 1, 1, 1, 1, # C
                                 1, 1, 2, 1, 1, 1, 1, 1, 1, 1, # C
                                 1, 1, 2, 1, 1, 1, 1, 1, 1, 2, # C
                                 1, 1, 1, 2, 1, 1, 1, 1, 1, 1, # D
                                 1, 1, 1, 2, 1, 2, 1, 1, 1, 1, # D
                                 1, 1, 1, 1, 2, 1, 1, 1, 1, 1, # E
                                 1, 1, 1, 1, 1, 2, 1, 1, 1, 1, # F
                                 1, 1, 1, 1, 1, 1, 2, 1, 1, 1, # G
                                 1, 1, 1, 1, 1, 1, 1, 2, 1, 1, # H
                                 1, 1, 1, 1, 1, 1, 1, 1, 2, 1, # I
                                 1, 1, 1, 1, 2, 1, 1, 1, 2, 1, # I
                                 1, 1, 1, 1, 1, 1, 1, 1, 1, 2),# J
                          dimnames=list(c("A", "A", "A", "A", "A", "A", "B", "B", "B", "C", "C", "C", "C",
                                          "D", "D", "E", "F", "G", "H", "I", "I", "J"), # row names
                                        c("A", "B", "C", "D", "E", "F", "G", "H", "I", "J")), # col names
                          byrow=T
  )
  
  # Compare each row of matrix to the observed page lengths
  template <- rownames(length_matrix)[
    which(
      apply(length_matrix, 1, function(row) all(row == page_lengths))
    )
  ]
  
  # TODO: Come up with a better method
  Etype2 <- F
  if(length(template) == 0){
    if(length(try(read_html(paste0(link, "/press-release")), silent=T))!=1){
      template <- "E"
      Etype2 < T
    }
  }
  
  if(debug) print(paste0("Template: ", template))
  
  # Throw an error if no rows match
  if(length(template) == 0) stop(paste0("Unknown template or bad URL for ", link)) 
  
  
  # With the template determined we can start scraping
  if(template=="A"){
    
    # Build the landing page link
    if(all(page_lengths==c(2, 1, 1, 2, 2, 2, 2, 1, 2, 2))) landing <- paste0(link, "/category/press-releases")
    else landing <- paste0(link, "/media/press-releases")
    
    # Do the scraping
    pressers <- rbind(pressers, 
                      page_scraper(page_index=0, link=landing, home=link, 
                                   start_date, end_date, template="A",
                                   debug=debug, page_limit=page_limit))
  }
  else if(template=="B"){
    
    # Build the landing page link
    landing <- paste0(link, "/news/documentquery.aspx?DocumentTypeID=27")
    
    # Do the scraping
    pressers <- rbind(pressers, 
                      page_scraper(page_index=0, link=landing, home=link, 
                                   start_date, end_date, template="B",
                                   debug=debug, page_limit=page_limit))
  }
  else if(template=="C"){
    # Build the landing page link
    landing <- paste0(link, "/category/congress_press_release")
    
    # Do the scraping
    pressers <- rbind(pressers, 
                      page_scraper(page_index=0, link=landing, home=link, 
                                   start_date, end_date, template="C",
                                   debug=debug, page_limit=page_limit))
  }
  else if(template=="D"){
    # Build the landing page link
    landing <- paste0(link, "/media-center/press-releases")
    
    # Do the scraping
    pressers <- rbind(pressers, 
                      page_scraper(page_index=0, link=landing, home=link, 
                                   start_date, end_date, template="D",
                                   debug=debug, page_limit=page_limit))
  }
  else if(template=="E"){
    # Build the landing page link
    landing <- ifelse(Etype2, paste0(link, "/press-release"), paste0(link, "/press-releases"))
    
    # Do the scraping
    pressers <- rbind(pressers, 
                      page_scraper(page_index=0, link=landing, home=link, 
                                   start_date, end_date, template="E",
                                   debug=debug, page_limit=page_limit))
  }
  else if(template=="F"){
  # Build the landing page link
  landing <- paste0(link, "/newsroom/press-releases")
  
  # Do the scraping
  pressers <- rbind(pressers, 
                    page_scraper(page_index=0, link=landing, home=link, 
                                 start_date, end_date, template="F",
                                 debug=debug, page_limit=page_limit))
  }
  else if(template=="G"){
  # Build the landing page link
  landing <- paste0(link, "/news/press-releases")
  
  # Do the scraping
  pressers <- rbind(pressers, 
                    page_scraper(page_index=0, link=landing, home=link, 
                                 start_date, end_date, template="G",
                                 debug=debug, page_limit=page_limit))
  }
  else if(template=="H"){
    # Build the landing page link
    landing <- paste0(link, "/category/press_release")
    
    # Do the scraping
    pressers <- rbind(pressers, 
                      page_scraper(page_index=0, link=landing, home=link, 
                                   start_date, end_date, template="H",
                                   debug=debug, page_limit=page_limit))
  }
  else if(template=="I"){
    # Build the landing page link
    landing <- paste0(link, "/category/press-releases")
    
    # Do the scraping
    pressers <- rbind(pressers, 
                      page_scraper(page_index=0, link=landing, home=link, 
                                   start_date, end_date, template="I",
                                   debug=debug, page_limit=page_limit))
  }
  else if(template=="J"){
    # Build the landing page link
    landing <- paste0(link, "/press")
    
    # Do the scraping
    pressers <- rbind(pressers, 
                      page_scraper(page_index=0, link=landing, home=link, 
                                   start_date, end_date, template="J",
                                   debug=debug, page_limit=page_limit))
  }
  
  row.names(pressers) <- NULL
  
  # Attach the legislator's name passed to the function and remove duplicates
  if(nrow(pressers)>0) pressers <- cbind(name=name, pressers) %>% unique()
  
  # Tell the user how many total press releases were found
  # (but don't overwrite previous messages when in debug mode)
  message(paste0(ifelse(debug, "", "\014"), nrow(pressers), 
                 " press releases found in date range for ", link, "."))
  
  return(pressers)
}
```

```{r}
# Example usage

## Template A
# scraper(name="Moore, Barry", link="https://barrymoore.house.gov",
#          start_date="2023-07-20",
#          end_date="2023-10-21") %>% View()

# scraper(link="https://biggs.house.gov",
#          start_date="2023-07-20",
#          end_date="2023-12-21") #%>% View()

# scraper(link="https://simon.house.gov",
#          start_date="2025-01-01",
#          end_date="2025-01-14") #%>% View()

## Template B
# scraper(link="https://gosar.house.gov",
#          start_date="2023-07-20",
#          end_date="2023-10-21") #%>% View()

# scraper(link="https://mikerogers.house.gov",
#          start_date="2023-07-20",
#          end_date="2023-10-21")

## Template C
# scraper(link="https://grijalva.house.gov",
#          start_date="2024-12-01",
#          end_date="2025-01-09")

# scraper(link="https://schweikert.house.gov", # Variation on C
#          start_date="2024-12-10",
#          end_date="2025-01-14", debug=T)

## Template D
# scraper(link="aderholt.house.gov",
#          start_date="2024-07-20",
#          end_date="2024-10-21")

## Template E
# scraper(link="https://sewell.house.gov/",
#          start_date="2024-07-20",
#          end_date="2024-10-21")

## Template F
# scraper(link="mikethompson.house.gov/",
#          start_date="2024-07-20",
#          end_date="2024-10-21")

# scraper(link="https://takano.house.gov", # type2 example
#          start_date="2024-12-01",
#          end_date="2024-12-31")

# scraper(link="mcclintock.house.gov",
#          start_date="2024-07-20",
#          end_date="2024-10-21")

## Template G
# scraper(link="pelosi.house.gov/",
#          start_date="2024-07-20",
#          end_date="2024-10-21")

## Template H
# scraper(link="kevinmullin.house.gov/",
#          start_date="2024-07-20",
#          end_date="2024-10-21")

## Template I
# scraper(link="https://juliabrownley.house.gov",
#          start_date="2024-12-01",
#          end_date="2024-12-31")

## Template J
# scraper(link="https://correa.house.gov",
#          start_date="2024-12-01",
#          end_date="2024-12-31")

## Unknown Template/Bad URL
# scraper(link="https://obama.house.gov",
#          start_date="2023-07-20",
#          end_date="2023-10-21") #%>% View()

```

# Getting list of all House member domains
```{r}
websites <- read_html("https://www.house.gov/representatives") %>% 
  html_nodes(".views-field-value-5") %>% 
  html_nodes('a') %>% html_attr('href') %>% 
  sapply(FUN=function(x) sub("(house\\.gov).*", "\\1", x)) # Clean junk tacked on the end

names <- read_html("https://www.house.gov/representatives") %>% 
  html_nodes(".views-field-value-5") %>% 
  html_nodes('a') %>%  html_text()
```

# Test case: scraping all members' press releases since December 1
```{r}
# Modify to log failed domains
# sapply(websites[which(!websites %in% c("https://crane.house.gov", 
#                                        "https://stanton.house.gov",
#                                        "https://hamadeh.house.gov",
#                                        "https://kiley.house.gov",
#                                        "https://mcclintock.house.gov",
#                                        "https://pelosi.house.gov",
#                                        "https://simon.house.gov",
#                                        "https://gray.house.gov",
#                                        "https://kevinmullin.house.gov"))], 
#        FUN=scraper, start_date="2024-12-01", end_date="2025-01-09")

# failed_URLs <- c()
# sapply(c(websites[1:3], "obama.house.gov"), 
#        FUN=function(x, start_date, end_date){
#          retVal <- try(scraper(x, start_date, end_date), silent=T)
#          if(class(retVal)=="error" | class(retVal)=="try-error") failed_URLs <- c(failed_URLs, x)
#          else retVal
#          }, start_date="2024-12-01", end_date="2025-01-09")

pressers <- data.frame()
failed_URLs <- c()
for(w in websites){
  p <- try(scraper(w, "2024-12-01", "2025-01-09"), silent=T)
  if(class(p)=="error" | class(p)=="try-error") failed_URLs <- c(failed_URLs, w)
  else pressers <- bind_rows(pressers, p)
}
```

```{r}
pressers <- data.frame()
failed_URLs2 <- c()
for(w in failed_URLs){
  p <- try(scraper(w, "2024-12-01", "2025-01-09", page_limit=20), silent=T)
  if(class(p)=="error" | class(p)=="try-error") failed_URLs2 <- c(failed_URLs2, w)
  else pressers <- bind_rows(pressers, p)
}

failed_URLs2
```


TODO:
- Date range not always respected, i.e. with Barry Moore template A
- Templates: 
  - crane.house.gov 
    - No pagination so likely need to go date by date appended to url
  - stanton.house.gov
    - should be template E but failing for some reason
  - https://hamadeh.house.gov
    - new member, not enough pressers to understand pagination
  - https://kiley.house.gov
    - Having trouble parsing to get date
    - Name: read_html(link) %>% html_nodes(xpath="/html/head/meta[3]") %>% html_attr("content")
      - Rather than "/html/head/meta[2]"
  - nikemawilliams

# Template examples

## Template A

https://aderholt.house.gov/media-center/press-releases

https://biggs.house.gov/media/press-releases

https://barrymoore.house.gov/media/press-releases

Stout code
```{r}
get_main = function(links){
  press_page=read_html(links)
  
  press_info=press_page %>% html_nodes(".evo-press-release__body") %>% 
    html_text() %>% str_trim
  
  return(press_info)
}


moorepr=data.frame()
state="AL"
district=2

for(page_result in seq(from = 0 , to = 15, by = 1))
{
  link = paste0("https://barrymoore.house.gov/media/press-releases?page=", page_result)
  
  page = read_html(link)
  
  name = page %>% html_nodes(".font-weight-bold a") %>% html_text()
  
  links = page %>% html_nodes(".font-weight-bold a") %>% html_attr("href") %>% 
    paste("https://barrymoore.house.gov", ., sep="")
  
  date = page %>% html_nodes(".col-auto:nth-child(1)") %>% html_text()
  
  prpage = sapply(links, FUN=get_main)
  
  moorepr = rbind(moorepr, data.frame(state, district, name, date, prpage))
  
  print(paste("Page:", page_result))  
}

write.csv(moorepr, "C:/Users/bestf/OneDrive/Desktop/Press Releases/mooreal2.csv")
```



## Template B

https://mikerogers.house.gov/news/documentquery.aspx?DocumentTypeID=27

https://gosar.house.gov/news/documentquery.aspx?DocumentTypeID=27

Stout code
```{r}
get_main=function(links) {
press_page=read_html(links)
press_info=press_page%>% html_nodes(".bodycopy") %>% html_text() %>%str_trim
return(press_info)}

gosarpr=data.frame()
state="AZ"
district=4

for(page_result in seq(from = 0, to = 15, by = 1))
{
link= paste0("https://gosar.house.gov/news/documentquery.aspx?DocumentTypeID=27&Page=", page_result)
page=read_html(link)



name=page %>% html_nodes(".newsie-titler a") %>% html_text()
links=page %>% html_nodes(".newsie-titler a") %>% html_attr("href") %>% paste("https://gosar.house.gov/news/", ., sep="")
date=page %>%html_nodes("time") %>% html_text()
prpage=sapply(links, FUN=get_main)

gosarpr=rbind(gosarpr, data.frame(state, district, name, date, prpage))
 
}

 write.csv(gosarpr, "C:/Users/bestf/OneDrive/Desktop/Press Releases/gosarpraz4.csv")
```


## Template C

https://grijalva.house.gov/category/congress_press_release/

schweikert.house.gov - variation

## Template D

https://aderholt.house.gov

## Template E

https://sewell.house.gov/


## Template F
mikethompson.house.gov

## Template G
pelosi.house.gov

# Scraper v2

```{r}
presser_finder <- function(link, debug=F){
  page <- read_html(link)
  
  # XPath explanation:
  # - //a selects all <a> elements.
  # - translate(., 'ABCDEFGHIJKLMNOPQRSTUVWXYZ', 'abcdefghijklmnopqrstuvwxyz') converts the text to lowercase.
  # - The first condition checks if the text contains "press release".
  # - The second condition checks if the text contains "news" but not "newsletter".
  # - The and @href part ensures that only nodes with an href attribute are selected.
  link_node <- html_node(
    page,
    xpath = "//a[
    (
      contains(translate(., 'ABCDEFGHIJKLMNOPQRSTUVWXYZ', 'abcdefghijklmnopqrstuvwxyz'), 'press release')
      or
      (
        contains(translate(., 'ABCDEFGHIJKLMNOPQRSTUVWXYZ', 'abcdefghijklmnopqrstuvwxyz'), 'news')
        and
        not(contains(translate(., 'ABCDEFGHIJKLMNOPQRSTUVWXYZ', 'abcdefghijklmnopqrstuvwxyz'), 'newsletter'))
      )
    )
    and @href
  ]"
  )
  
  if (!is.null(link_node)) {
    href_value <- html_attr(link_node, "href")
  } else {
    message("No relevant link found.")
  }
  
  return(href_value)
}

scraper <- function(link, start_date, end_date, name=NA, page_limit=1000, debug=F){
  
  # Check if link has https:// at the start, if not add it
  if(substr(link, 1, 8)!="https://") link <- paste0("https://", link)
  
  # Check if link is in valid format and date range is valid
  if(!grepl("https://[a-zA-Z0-9\\-]+\\.house\\.gov", link)){
    stop("Link is not of the form \"{name}.house.gov\" or \"https://{name}.house.gov\"")
  }
  
  if(start_date > end_date){
    stop("Start date is after end date")
  }
  
  # Sanitize input by removing extra forward slash at end of URL if present
  if(substr(link, nchar(link), nchar(link))=="/") link <- substr(link, 1, nchar(link)-1)
  
  # Data frame that will hold scraped pressers
  pressers <- data.frame(date=c(), title=c(), body=c(), url=c())
  
  # Find the link for the press release page
  pressers_link <- presser_finder(link, debug=debug)
  
  if(is.na(link)) stop(paste0("Press release page could not be found for ", link))
  
  # Build the URL for the press releases page
  if (!grepl("^https?://", pressers_link)) {
    # Remove trailing slashes from link
    base_link <- sub("/+$", "", link)
    
    # Remove leading slashes from pressers_link
    relative_link <- sub("^/+", "", pressers_link)
    
    # Combine with a single slash between them
    pressers_link <- paste0(base_link, "/", relative_link)
  }
  
  # Read in our webpage elements to look through
  nodes <- read_excel("nodes.xlsx")
  
  pagination <- nodes %>% filter(element=="page link")
  
  page_index_start <- NA
  page_string <- NA
  
  # Loop through press release page link formats until one works, then determine
  # if the pagination starts at 0 or 1
  for(p in pagination$string){
    page0 <- try(read_html(paste0(pressers_link, p, 0)))
    page1 <- try(read_html(paste0(pressers_link, p, 1)))
    
    # A non-blank page will have length > 1
    if(length(page0)==1 & length(page0)==1) next
    else if(length(page1)==2){ page_index_start <- 1; page_string <- p; break }
    else if(length(page0)==2){ page_index_start <- 0; page_string <- p; break }
  }
  
  if(is.na(page_string)) stop(paste0("Pagination failed for ", link))
  
  # Build the press release page URL
  pressers_link <- paste0(pressers_link, page_string, page_index_start)
  
  date_nodes <- nodes %>% filter(element=="dates")
  
  date_node <- NA
  date_format <- NA
  
  # Loop through date notes until one works
  for(i in 1:length(date_nodes)){
    
    # Case when date can be found with xpath
    if(date_nodes$type[i]=="xpath"){
      date_attempt <- try(read_html(pressers_link) %>% 
                          html_nodes(xpath=date_nodes$string[i]) %>% html_text() %>% str_trim())
    }
    # Case when date can be found with CSS
    else{
      date_attempt <- try(read_html(pressers_link) %>% 
                            html_nodes(date_nodes$string[i]) %>% html_text() %>% str_trim())
    }
    if(length(date_attempt)==1) next
    else{
      date_format <- detect_date_format(date_attempt)
      
      print(date_format)
      
      if(is.na(date_format)) next
      else date_node <- date_nodes$string[i]; break 
    }
  }
  
  if(is.na(date_format)) stop(paste0("No valid date node found for ", link))
  
  # TODO: Check format of dates
  
  # TODO: Check format of titles
  
  # TODO: Check format of pr links
  
  # TODO: Check format of texts
  
  # TODO: Pass formats to recursive helper function
  
  # TODO: Return the pressers
  
  return(pressers_link)
  
}

text_scraper <- function(page_index, link, home, start_date, end_date,
                         date_node, date_node_type,
                         title_node, title_node_type,
                         pr_links_node, pr_links_node_type,
                         text_node, text_node_type,
                         page_string, clean_text=F, # TODO: add option to have clean and raw text
                         debug=F, page_limit=500){}

scraper(link="barrymoore.house.gov", start_date="2024-12-01", end_date="2024-12-31", name=NA, page_limit=1000, debug=T)
```

```{r}
# No date nodes working, .col-auto:nth-child(1) only gives one date
scraper(link="barrymoore.house.gov", start_date="2024-12-01", end_date="2024-12-31", name=NA, page_limit=1000, debug=T)


read_html("https://barrymoore.house.gov/media/press-releases?page=1") %>% html_nodes(".col-auto:nth-child(1)") %>% html_text() %>% str_trim()

```

"September 11, 2024" "September 10, 2024" "August 28, 2024"