文章以文件形式存储

Author: MrHangVIP

baike_bug/bug_main.py

@@ -1,9 +1,16 @@
 # coding:utf8
-from baike_bug import url_manager, html_downloader, html_parser, html_output, db_config, db_util
+import sys
+
+from baike_bug import url_manager, html_downloader, html_parser, html_output, db_util
+
+defaultencoding = 'utf-8'
 
 
 class BugMain(object):
     def __init__(self):
+        if sys.getdefaultencoding() != defaultencoding:
+            reload(sys)
+            sys.setdefaultencoding(defaultencoding)
         self.urls = url_manager.UrlManager()
         self.downloader = html_downloader.HtmlDownloader()
         self.parser = html_parser.HtmlParser()
@@ -23,29 +30,32 @@ def craw(self, root_url):
                 if info_data is not None:
                     sql = "insert into t_novel(novelurl, novelname, clicknum, wordsnum, type, " \
                           "author, isfinish, biref, imageurl) values " \
-                          "('%s', '%s', '%s', '%s', '%s', '%s', '%s', '%s', '%s')" %\
-                          (info_data['info_url'] ,info_data['novelName'], info_data['clickNum'],
-                           info_data['wordsNum'], info_data['type'], info_data['author'], info_data['state'],
-                           info_data['brief'], info_data['imageUrl'])
+                          "('%s', '%s', '%s', '%s', '%s', '%s', '%s', '%s', '%s')" % \
+                          (info_data['info_url'].encode('utf-8'), info_data['novelName'].encode('utf-8'),
+                           info_data['clickNum'],
+                           info_data['wordsNum'].encode('utf-8'), info_data['type'].encode('utf-8'),
+                           info_data['author'].encode('utf-8'), info_data['state'].encode('utf-8'),
+                           info_data['brief'].encode('utf-8'), info_data['imageUrl'].encode('utf-8'))
                     self.dbutil.insert(sql)
 
                 if chapter_datas is not None:
                     for chapter_data in chapter_datas:
                         sql = "insert into t_chapter(novelurl, chaptername, chapterurl, chapternum) values " \
                               "('%s', '%s', '%s', '%s')" % \
-                              (chapter_data['info_url'], chapter_data['chapterName'], chapter_data['chapterUrl'],
-                               chapter_data['chapterNum'])
+                              (chapter_data['info_url'].encode('utf-8'), chapter_data['chapterName'].encode('utf-8'),
+                               chapter_data['chapterUrl'].encode('utf-8'),
+                               chapter_data['chapterNum'].encode('utf-8'))
                         self.dbutil.insert(sql)
                 # if count == 2:
                 #     break
                 count = count + 1
-            except:
-                print 'craw fail'
+            except Exception, e:
+                print 'craw fail:' + str(e)
         print 'finish'
         # 关闭数据库连接
         self.dbutil.close()
-            # self.outputer.output_html()
-        self.dbutil.db.close()#关闭数据库链接
+        # self.outputer.output_html()
+        self.dbutil.db.close()  # 关闭数据库链接
 
 
 if __name__ == "__main__":

baike_bug/db_config.py

@@ -4,5 +4,5 @@
 
 class DB_Config(object):
     def connect(self):
-        conn = MySQLdb.connect(host="localhost", user="root", passwd="123456", db="noveldb", charset="utf8")
+        conn = MySQLdb.connect(host="127.0.0.1", user="root", passwd="12345", db="noveldb", charset="utf8")
         return conn

baike_bug/db_util.py

@@ -1,9 +1,16 @@
 # encoding:utf8
+import sys
+
 from baike_bug import db_config
 
+defaultencoding = 'utf-8'
+
 
 class DB_Util(object):
     def __init__(self):
+        if sys.getdefaultencoding() != defaultencoding:
+            reload(sys)
+            sys.setdefaultencoding(defaultencoding)
         self.db = db_config.DB_Config().connect()
         self.cursor = self.db.cursor()
 
@@ -13,8 +20,9 @@ def insert(self, sql):
             self.cursor.execute(sql)
             # 提交到数据库执行
             self.db.commit()
-        except:
+        except Exception, e:
             # 发生错误时回滚
+            print "insert fail:" + e
             self.db.rollback()
 
     def query(self, sql):
@@ -38,6 +46,3 @@ def query(self, sql):
                       (fname, lname, age, sex, income)
         except:
             print "Error: unable to fecth data"
-
-
-

baike_bug/file_output.py

@@ -0,0 +1,20 @@
+# coding:utf8
+import os
+
+
+class FileOutPut(object):
+    def __init__(self):
+        self.data = {}
+
+    # 参数 文件数据，文件夹名称，文件名称
+    def file_output(self, data, dirname, filename):
+        filepath = r'../novelfile/%s/' % dirname
+        if os.path.exists(filepath) is False:
+            os.mkdir(filepath)
+        try:
+            filehandle = open(filepath + filename + ".txt", "w")
+            filehandle.write(data.encode("utf-8"))
+            filehandle.close()
+        except Exception, e:
+            print e
+        return filepath + filename + ".txt"

baike_bug/html_parser.py

@@ -2,15 +2,17 @@
 import re
 import urlparse
 
+from baike_bug import html_downloader, db_util
 from bs4 import BeautifulSoup
 
-from baike_bug import html_downloader, db_util
+from baike_bug import file_output
 
 
 class HtmlParser(object):
     def __init__(self):
         self.downloader = html_downloader.HtmlDownloader()
         self.dbutil = db_util.DB_Util()
+        self.outputFile = file_output.FileOutPut()
 
     # 本地方法需要先定义在使用，也就是定义得放在前面不然无法调用
     def _get_new_urls(self, page_url, soup):
@@ -41,20 +43,20 @@ def _get_new_data(self, page_url, soup):
 
         # 文章数据解析
 
-    def _parse_chapter(self, chapter_url):
+    def _parse_chapter(self, chapter_url, info_url):
         if chapter_url is None:
             return
         html_cont = self.downloader.download(chapter_url)
         if html_cont is None:
             return
         soup = BeautifulSoup(html_cont, 'html.parser', from_encoding='utf-8')
         # new_urls = self._get_new_urls(chapter_url, soup)  # self调本地方法，通过url
-        content_data = self._parse_content_data(chapter_url, soup)
+        content_data = self._parse_content_data(info_url, chapter_url, soup)
 
         if content_data is not None:
             sql = "insert into t_content(chapterurl, content, nexturl, preurl) values " \
                   "('%s', '%s', '%s', '%s')" % \
-                  (content_data['chapterUrl'], content_data['content'], content_data['nextUrl'],
+                  (content_data['chapterUrl'], content_data['content'].encode('utf-8'), content_data['nextUrl'],
                    content_data['preUrl'])
             self.dbutil.insert(sql)
 
@@ -110,35 +112,40 @@ def _parse_chapter_data(self, page_url, soup):
         chapter_list = []
         if page_url.find("info") == -1:
             return None
-        count = 0
+        count = 1
         # 匹配title  <dd class="lemmaWgt-lemmaTitle-title">
         chapter_list_node = soup.find('div', class_="volume-wrap")
         for chapter in chapter_list_node.find_all('li'):
+            if count > 1:
+                return chapter_list
             chapter_data = {}
             chapter_data['info_url'] = page_url
             chapter_data['chapterNum'] = count
             chapter_data['chapterName'] = chapter.get_text()
             chapter_data['chapterUrl'] = chapter.find('a').get('href')
             # join 方法会按照pageurl的格式将new_url补全
             chapter_data['chapterUrl'] = urlparse.urljoin(page_url, chapter_data['chapterUrl'])
+            count = count + 1
             try:
                 if chapter.find('a').get('href') is not None:
-                    self._parse_chapter(chapter_data['chapterUrl'])  # 解析内容数据
-            except:
-                print "parse_chapter() fail "
+                    self._parse_chapter(chapter_data['chapterUrl'], page_url)  # 解析内容数据
+            except Exception, e:
+                print "parse_chapter() fail:" + e
             chapter_list.append(chapter_data)
-        return chapter_data
+        return chapter_list
 
     # 内容信息解析  文章内容，章节url，上一章节url，下一章节url
-    def _parse_content_data(self, page_url, soup):
+    def _parse_content_data(self, info_url, chapter_url, soup):
         # self._get_new_urls(page_url, soup)
         content_data = {}
-        if page_url.find("chapter") == -1:
+        if chapter_url.find("chapter") == -1:
             return None
-        content_data['chapterUrl'] = page_url
+        content_data['chapterUrl'] = chapter_url
         content_node = soup.find('div', class_="read-content j_readContent")
         # content_data['content'] = content_node.get_text()
-        content_data['content'] = "hahah"
+        filepath = self.outputFile.file_output(content_node.get_text(), info_url.split("/").pop(),
+                                               chapter_url.split("/").pop())
+        content_data['content'] = filepath
         # p_nodes = content_node.find_all('p')
         # for content in p_nodes:
         #     content_data['content'] = content_data['content'] + '\\n' + content.get_text()
@@ -148,10 +155,10 @@ def _parse_content_data(self, page_url, soup):
         chapter_node = soup.find('div', class_="chapter-control dib-wrap")
         for a in chapter_node.find_all('a'):
             if a.get('id') is not None and a.get('id').find("j_chapterPrev") != -1:
-                content_data['preUrl'] = a.get('href')
+                content_data['preUrl'] = urlparse.urljoin(chapter_url, a.get('href'))
                 continue
             if a.get('id') is not None and a.get('id').find("j_chapterNext") != -1:
-                content_data['nextUrl'] = a.get('href')
+                content_data['nextUrl'] = urlparse.urljoin(chapter_url, a.get('href'))
                 continue
         return content_data