fixing except block directly handling baseException

Author: slowy07

news_articles__scraper.py

@@ -11,61 +11,64 @@
 
 # ! pip install newspaper3k
 
-# importing necessary libraries
-from bs4 import BeautifulSoup
-import requests
-import urllib
-import pandas as pd
-from newspaper import Article
 import pickle
 import re
 import sys
+import urllib
+
+import pandas as pd
+import requests
+
+# importing necessary libraries
+from bs4 import BeautifulSoup
+from newspaper import Article
 
 # Extracting links for all the pages (1 to 158) of boomlive fake news section
 fakearticle_links = []
 for i in range(1, 159):
-  url = 'https://www.boomlive.in/fake-news/' + str(i)
-  try:
-    # this might throw an exception if something goes wrong.
-    page=requests.get(url) 
-
-    # send requests
-    page = requests.get(url)
-    soup = BeautifulSoup(page.text, 'html.parser')  
-
-    # Collecting all the links in a list
-    for content in soup.find_all('h2', attrs={'class':'entry-title'}):
-      link = content.find('a')
-      fakearticle_links.append(link.get('href')) 
-  
-  # this describes what to do if an exception is thrown 
-  except Exception as e:    
-    # get the exception information
-    error_type, error_obj, error_info = sys.exc_info()      
-    #print the link that cause the problem
-    print ('ERROR FOR LINK:',url)
-    #print error info and line that threw the exception                          
-    print (error_type, 'Line:', error_info.tb_lineno)
-    continue
+    url = "https://www.boomlive.in/fake-news/" + str(i)
+    try:
+        # this might throw an exception if something goes wrong.
+        page = requests.get(url)
+
+        # send requests
+        page = requests.get(url)
+        soup = BeautifulSoup(page.text, "html.parser")
+
+        # Collecting all the links in a list
+        for content in soup.find_all("h2", attrs={"class": "entry-title"}):
+            link = content.find("a")
+            fakearticle_links.append(link.get("href"))
+
+    # this describes what to do if an exception is thrown
+    except Exception as e:
+        # get the exception information
+        error_type, error_obj, error_info = sys.exc_info()
+        # print the link that cause the problem
+        print("ERROR FOR LINK:", url)
+        # print error info and line that threw the exception
+        print(error_type, "Line:", error_info.tb_lineno)
+        continue
 
 fakearticle_links[:5]
 
 len(fakearticle_links)
 
 fakearticle_links[1888:]
 
+import matplotlib.pyplot as plt
 import pandas as pd
+
 import numpy as np
-import matplotlib.pyplot as plt
 
 """We have to modify the links so that the links actually work as we can see that the string extracted is the last part of the url!
 
 **We have to add 'https://www.boomlive.in/fake-news' to the extracted links.**
 """
 
 # Modify the links so that it takes us to the particular website
-str1 = 'https://www.boomlive.in/fake-news'
-fakearticle_links = [str1+lnk for lnk in fakearticle_links]
+str1 = "https://www.boomlive.in/fake-news"
+fakearticle_links = [str1 + lnk for lnk in fakearticle_links]
 
 fakearticle_links[6:10]
 
@@ -75,30 +78,37 @@
 """
 
 # Create a dataset for storing the news articles
-news_dataset = pd.DataFrame(fakearticle_links, columns=['URL'])
+news_dataset = pd.DataFrame(fakearticle_links, columns=["URL"])
 
 news_dataset.head()
 
-title, text, summary, keywords, published_on, author = [], [], [], [], [], []   # Creating empty lists to store the data
+title, text, summary, keywords, published_on, author = (
+    [],
+    [],
+    [],
+    [],
+    [],
+    [],
+)  # Creating empty lists to store the data
 for Url in fakearticle_links:
-  article = Article(Url)
-
-  #Call the download and parse methods to download information
-  try:
-    article.download()
-    article.parse()
-    article.nlp()
-  except Exception as error:
-    print(f"exception : {error}")
-    pass
-  
-  # Scrape the contents of article
-  title.append(article.title)                    # extracts the title of the article
-  text.append(article.text)                      # extracts the whole text of article
-  summary.append(article.summary)                # gives us a summary abou the article
-  keywords.append(', '.join(article.keywords))   # the main keywords used in it
-  published_on.append(article.publish_date)      # the date on which it was published
-  author.append(article.authors)                 # the authors of the article
+    article = Article(Url)
+
+    # Call the download and parse methods to download information
+    try:
+        article.download()
+        article.parse()
+        article.nlp()
+    except Exception as error:
+        print(f"exception : {error}")
+        pass
+
+    # Scrape the contents of article
+    title.append(article.title)  # extracts the title of the article
+    text.append(article.text)  # extracts the whole text of article
+    summary.append(article.summary)  # gives us a summary abou the article
+    keywords.append(", ".join(article.keywords))  # the main keywords used in it
+    published_on.append(article.publish_date)  # the date on which it was published
+    author.append(article.authors)  # the authors of the article
 
 """**Checking the lists created**"""
 
@@ -111,107 +121,118 @@
 author[6]
 
 # Adding the columns in the fake news dataset
-news_dataset['title'] = title
-news_dataset['text'] = text
-news_dataset['keywords'] = keywords
-news_dataset['published date'] = published_on
-news_dataset['author'] = author
+news_dataset["title"] = title
+news_dataset["text"] = text
+news_dataset["keywords"] = keywords
+news_dataset["published date"] = published_on
+news_dataset["author"] = author
 
 # Check the first five columns of dataset created
 news_dataset.head()
 
 """**Converting the dataset to a csv file**"""
 
-news_dataset.to_csv('Fake_news.csv')
+news_dataset.to_csv("Fake_news.csv")
 
 """**Reading the csv file**"""
 
-df = pd.read_csv('Fake_news.csv')
+df = pd.read_csv("Fake_news.csv")
 
 # Checking the last 5 rows of the csv file
 df.tail(5)
 
 """**Download the csv file in local machine**"""
 
 from google.colab import files
-files.download('Fake_news.csv')
+
+files.download("Fake_news.csv")
 
 """**Scraping news from Times of India**"""
 
-TOIarticle_links = []  # Creating an empty list of all the urls of news from Times of India site
+TOIarticle_links = (
+    []
+)  # Creating an empty list of all the urls of news from Times of India site
 
 # Extracting links for all the pages (2 to 125) of boomlive fake news section
 for i in range(2, 126):
-  url = 'https://timesofindia.indiatimes.com/news/' + str(i)
-  
-  try:
-    # send requests
-    page = requests.get(url)
-    soup = BeautifulSoup(page.text, 'html.parser')  
-
-    # Collecting all the links in a list
-    for content in soup.find_all('span', attrs={'class':'w_tle'}):
-      link = content.find('a')
-      TOIarticle_links.append(link.get('href')) 
-  
-  # this describes what to do if an exception is thrown 
-  except Exception as e:    
-    # get the exception information
-    error_type, error_obj, error_info = sys.exc_info()      
-    #print the link that cause the problem
-    print ('ERROR FOR LINK:',url)
-    #print error info and line that threw the exception                          
-    print (error_type, 'Line:', error_info.tb_lineno)
-    continue
+    url = "https://timesofindia.indiatimes.com/news/" + str(i)
+
+    try:
+        # send requests
+        page = requests.get(url)
+        soup = BeautifulSoup(page.text, "html.parser")
+
+        # Collecting all the links in a list
+        for content in soup.find_all("span", attrs={"class": "w_tle"}):
+            link = content.find("a")
+            TOIarticle_links.append(link.get("href"))
+
+    # this describes what to do if an exception is thrown
+    except Exception as e:
+        # get the exception information
+        error_type, error_obj, error_info = sys.exc_info()
+        # print the link that cause the problem
+        print("ERROR FOR LINK:", url)
+        # print error info and line that threw the exception
+        print(error_type, "Line:", error_info.tb_lineno)
+        continue
 
 TOIarticle_links[6:15]
 
 len(TOIarticle_links)
 
-str2 = 'https://timesofindia.indiatimes.com'
-TOIarticle_links = [str2+lnk for lnk in TOIarticle_links if lnk[0]=='/']
+str2 = "https://timesofindia.indiatimes.com"
+TOIarticle_links = [str2 + lnk for lnk in TOIarticle_links if lnk[0] == "/"]
 
 TOIarticle_links[5:8]
 
 len(TOIarticle_links)
 
-title, text, summary, keywords, published_on, author = [], [], [], [], [], []   # Creating empty lists to store the data
+title, text, summary, keywords, published_on, author = (
+    [],
+    [],
+    [],
+    [],
+    [],
+    [],
+)  # Creating empty lists to store the data
 for Url in TOIarticle_links:
-  article = Article(Url)
-
-  #Call the download and parse methods to download information
-  try:
-    article.download()
-    article.parse()
-    article.nlp()
-  except:
-    pass
-  
-  # Scrape the contents of article
-  title.append(article.title)                    # extracts the title of the article
-  text.append(article.text)                      # extracts the whole text of article
-  summary.append(article.summary)                # gives us a summary abou the article
-  keywords.append(', '.join(article.keywords))   # the main keywords used in it
-  published_on.append(article.publish_date)      # the date on which it was published
-  author.append(article.authors)                 # the authors of the article
+    article = Article(Url)
+
+    # Call the download and parse methods to download information
+    try:
+        article.download()
+        article.parse()
+        article.nlp()
+    except Exception:
+        pass
+
+    # Scrape the contents of article
+    title.append(article.title)  # extracts the title of the article
+    text.append(article.text)  # extracts the whole text of article
+    summary.append(article.summary)  # gives us a summary abou the article
+    keywords.append(", ".join(article.keywords))  # the main keywords used in it
+    published_on.append(article.publish_date)  # the date on which it was published
+    author.append(article.authors)  # the authors of the article
 
 title[5]
 
-TOI_dataset = pd.DataFrame(TOIarticle_links, columns=['URL'])
+TOI_dataset = pd.DataFrame(TOIarticle_links, columns=["URL"])
 # Adding the columns in the TOI news dataset
-TOI_dataset['title'] = title
-TOI_dataset['text'] = text
-TOI_dataset['keywords'] = keywords
-TOI_dataset['published date'] = published_on
-TOI_dataset['author'] = author
+TOI_dataset["title"] = title
+TOI_dataset["text"] = text
+TOI_dataset["keywords"] = keywords
+TOI_dataset["published date"] = published_on
+TOI_dataset["author"] = author
 
 TOI_dataset.head()
 
-TOI_dataset.to_csv('TOI_news_dataset.csv')
+TOI_dataset.to_csv("TOI_news_dataset.csv")
 
-dt = pd.read_csv('TOI_news_dataset.csv')
+dt = pd.read_csv("TOI_news_dataset.csv")
 
 dt.tail(3)
 
 from google.colab import files
-files.download('TOI_news_dataset.csv')
+
+files.download("TOI_news_dataset.csv")