From c82654153847d9920f1ce5cdc5c7a926ef2b092e Mon Sep 17 00:00:00 2001 From: Peterorr Date: Wed, 4 Aug 2021 18:09:09 +0800 Subject: [PATCH 1/2] pipenv built; refactor scraper as a class --- baseball_lineup_analysis/src/data/scraper.py | 60 ++++++++++++++++++++ 1 file changed, 60 insertions(+) create mode 100644 baseball_lineup_analysis/src/data/scraper.py diff --git a/baseball_lineup_analysis/src/data/scraper.py b/baseball_lineup_analysis/src/data/scraper.py new file mode 100644 index 0000000..1851883 --- /dev/null +++ b/baseball_lineup_analysis/src/data/scraper.py @@ -0,0 +1,60 @@ +import os +import re +import time + +import requests +from bs4 import BeautifulSoup +from selenium import webdriver +from selenium.webdriver.chrome.options import Options + + +class BaseballGameStatistics: + + def __init__(self, path): + self.driver = webdriver.Chrome(executable_path=os.path.abspath(path)) + self.inning_numbers = [] + self.base_soup = BeautifulSoup() + self.inning_soups = {} + + def game_request(self, year, kind_code, game_sno): + self.driver.get(f"https://www.cpbl.com.tw/box/live?year={year}&KindCode={kind_code}&gameSno={str(game_sno)}") + self.base_soup = BeautifulSoup(self.driver.page_source, "html.parser") + + def get_inning_numbers(self): + inning_numbers = [ + inning.get_text() + for inning in self.base_soup.find("div", {"class" : "InningPlaysGroup"}).\ + find("div", {"class" : "tabs"}).findAll("li") + ] + self.inning_numbers = inning_numbers + + def click_inning(self, inning_num): + click_path = \ + f"/html/body/div[2]/div[1]/div/form/div[4]/div[2]/div[2]/div[2]/div/div[1]/ul/li[{str(inning_num)}]/a/span" + self.driver.find_element_by_xpath(click_path).click() + self.inning_soups[inning_num] = {"all": BeautifulSoup(self.driver.page_source, "html.parser")} + + def all_plays_half_inning_parser(self, inning_num, half_inning): + half_inning_soup = self.inning_soups[inning_num]["all"].find("div", {"class" : "tab_cont all_plays active"}).\ + find("div",{"class" : "InningPlaysGroup"}).\ + find("section", {"class" : half_inning}).\ + findAll("div",{"class","item play"}) + self.inning_soups[inning_num][half_inning] = {"soup_list": half_inning_soup} + + def get_player_names(self, inning_num, half_inning): + player_names = [soup.find("div", {"class" : "info"}).\ + find("div", {"class" : "desc"}).find("a").get_text() + for soup in self.inning_soups[inning_num][half_inning]["soup_list"] + ] + self.inning_soups[inning_num][half_inning]["player_names"] = player_names + + # def get_pitch_counts(soup, status): + # pitches = soup.find("div", {"class","info"}).\ + # find("div", {"class", status}).\ + # findAll("span", {"class", re.compile("\s+active$")}) + # return len(pitches) + + # def get_batting_result(soup): + # result = soup.find("div", {"class","info"}). \ + # find("div", {"class", re.compile("batter_event\s+")}).get_text() + # return result From 25a610e0d8340a2f7798d7c11323923112c3e4e7 Mon Sep 17 00:00:00 2001 From: Peterorr Date: Wed, 11 Aug 2021 00:20:56 +0800 Subject: [PATCH 2/2] refactor scraper and confirm inning soups --- baseball_lineup_analysis/src/data/scraper.py | 69 ++++++++++---------- 1 file changed, 35 insertions(+), 34 deletions(-) diff --git a/baseball_lineup_analysis/src/data/scraper.py b/baseball_lineup_analysis/src/data/scraper.py index 1851883..e574035 100644 --- a/baseball_lineup_analysis/src/data/scraper.py +++ b/baseball_lineup_analysis/src/data/scraper.py @@ -1,3 +1,4 @@ +import logging import os import re import time @@ -6,19 +7,28 @@ from bs4 import BeautifulSoup from selenium import webdriver from selenium.webdriver.chrome.options import Options +from selenium.webdriver.common.action_chains import ActionChains +from selenium.webdriver.common.keys import Keys class BaseballGameStatistics: def __init__(self, path): self.driver = webdriver.Chrome(executable_path=os.path.abspath(path)) - self.inning_numbers = [] + self.driver.implicitly_wait(1) self.base_soup = BeautifulSoup() + self.score_soup = BeautifulSoup() + self.inning_numbers = [] self.inning_soups = {} - def game_request(self, year, kind_code, game_sno): - self.driver.get(f"https://www.cpbl.com.tw/box/live?year={year}&KindCode={kind_code}&gameSno={str(game_sno)}") + def get_base_soup(self, year, kind_code, game_sno): + self.driver.get( + f"https://www.cpbl.com.tw/box/live?year={year}&KindCode={kind_code}&gameSno={str(game_sno)}" + ) self.base_soup = BeautifulSoup(self.driver.page_source, "html.parser") + + def get_score_soup(self): + self.score_soup = self.base_soup.find("div", {"class" : "tab_cont scoring active"}) def get_inning_numbers(self): inning_numbers = [ @@ -27,34 +37,25 @@ def get_inning_numbers(self): find("div", {"class" : "tabs"}).findAll("li") ] self.inning_numbers = inning_numbers - - def click_inning(self, inning_num): - click_path = \ - f"/html/body/div[2]/div[1]/div/form/div[4]/div[2]/div[2]/div[2]/div/div[1]/ul/li[{str(inning_num)}]/a/span" - self.driver.find_element_by_xpath(click_path).click() - self.inning_soups[inning_num] = {"all": BeautifulSoup(self.driver.page_source, "html.parser")} - - def all_plays_half_inning_parser(self, inning_num, half_inning): - half_inning_soup = self.inning_soups[inning_num]["all"].find("div", {"class" : "tab_cont all_plays active"}).\ - find("div",{"class" : "InningPlaysGroup"}).\ - find("section", {"class" : half_inning}).\ - findAll("div",{"class","item play"}) - self.inning_soups[inning_num][half_inning] = {"soup_list": half_inning_soup} - - def get_player_names(self, inning_num, half_inning): - player_names = [soup.find("div", {"class" : "info"}).\ - find("div", {"class" : "desc"}).find("a").get_text() - for soup in self.inning_soups[inning_num][half_inning]["soup_list"] - ] - self.inning_soups[inning_num][half_inning]["player_names"] = player_names - - # def get_pitch_counts(soup, status): - # pitches = soup.find("div", {"class","info"}).\ - # find("div", {"class", status}).\ - # findAll("span", {"class", re.compile("\s+active$")}) - # return len(pitches) - - # def get_batting_result(soup): - # result = soup.find("div", {"class","info"}). \ - # find("div", {"class", re.compile("batter_event\s+")}).get_text() - # return result + for num in self.inning_numbers: + self.inning_soups[num] = {} + + def get_play_by_play(self, inning_num): + click_path = f"//li[a[span[contains(text(), {str(inning_num)})]]]" + element = self.driver.find_element_by_xpath(click_path) + element.click() + self.inning_soups[inning_num]["play_by_play"] = BeautifulSoup(self.driver.page_source, "html.parser") + + def get_pitch_by_pitch(self, inning_num): + click_path = f"//li[a[span[contains(text(),{str(inning_num)})]]]" + element = self.driver.find_element_by_xpath(click_path) + element.click() + click_path = "//div[@class='tab_cont all_plays active']//*[contains(@class, 'batter_event') or contains(@class, 'no-pitch-action-remind')]" + elements = self.driver.find_elements_by_xpath(click_path) + for index, element in enumerate(elements): + time.sleep(0.5) + try: + element.click() + except: + logging.info(f"element {index} is not clickable at point") + self.inning_soups[inning_num]["pitch_by_pitch"] = BeautifulSoup(self.driver.page_source, "html.parser")