time:2014-5-29 23:15:45

update the video_detail_crwaler.py
           app.py
           video_list_crawler.py
commit new file :
           DAO.py

Author: wuhan419

DAO.py

@@ -0,0 +1,69 @@
+__author__ = 'wuhan'
+import mysql.connector
+import copy
+
+class DAO(object):
+    def __init__(self):
+        self.__user = 'avoper'
+        self.__pwd = '******'
+        self.__db_host = '127.0.0.1'
+        self.__db = 'avdb'
+
+    def execute_dml(self, sql):
+
+        """
+    execute sql update and insert
+    :param sql:
+    :return:
+    """
+
+        cnx = mysql.connector.connect(user=self.__user, password=self.__pwd, host=self.__db_host, database=self.__db)
+        print(sql)
+        cursor = cnx.cursor()
+        try:
+            cursor.execute(sql)
+        except mysql.connector.Error as sql_err:
+            print("Error: {}".format(sql_err.msg))
+            log_sql = open('test.log', 'a')
+            log_sql.write("Error: {} \n in the insert/update sql :{}".format(sql_err.msg, sql))
+            log_sql.close()
+        cnx.commit()
+        cursor.close()
+        cnx.close
+
+
+    def execute_query(self, sql):
+        """
+        execute sql query
+        :param sql:
+        :return: result_list
+        """
+
+        cnx = mysql.connector.connect(user=self.__user, password=self.__pwd, host=self.__db_host, database=self.__db)
+        print(sql)
+        cursor = cnx.cursor()
+        try:
+            cursor.execute(sql)
+            result_rows = copy.deepcopy(cursor.fetchone())
+            if result_rows:
+                print(result_rows)
+                cursor.close()
+                cnx.close
+                return result_rows
+            if result_rows is None:
+                print("reuslt is null")
+                cursor.close()
+                cnx.close
+                return None
+        except mysql.connector.Error as sql_err:
+            print("Error: {}".format(sql_err.msg))
+            log_sql = open('test.log', 'a')
+            log_sql.write("Error: {} \n in the insert/update sql :{}".format(sql_err.msg, sql))
+            log_sql.close()
+
+            #cursor.close()
+
+
+if __name__ == '__main__':
+    Dao = DAO()
+    Dao.execute_query("select * from av_tag")

app.py

@@ -2,9 +2,9 @@
 __author__ = 'Sean Lei'
 
 from video_list_crawler import VideoListCrawler
-
+from video_detail_crawler import *
 
 if __name__ == '__main__':
     crawler = VideoListCrawler()
     crawler.craw()
-    print(crawler.detail_info_urls)
+    ##crawlerdetal = VideoDetailCrawler()

video_detail_crawler.py

@@ -1,12 +1,96 @@
 # -*- coding: utf-8 -*-
-__author__ = 'Sean Lei'
+__author__ = 'Sean Lei&wuhan'
 
 from base_crawler import BaseCrawler
 from pyquery import PyQuery as Pq
-
+from DAO import DAO
 
 class VideoDetailCrawler(BaseCrawler):
+    def __init__(self, seed_url):
+        self._seed_url = seed_url
+        self.__video_detail = {
+            'id': '',
+            'name': '',
+            'url': "",
+            'img': '',
+            'maker': ''
+        }
+        self.__tags = []
+        self.__cast = []
+
+    def _visit_pages(self):
+        """
+        @override
+        in this class ,only one page
+        """
+        html = self.get_page_content_str(self._seed_url)
+        self._extract_data(html)
+
     def _extract_data(self, doc_str):
         doc = Pq(doc_str)
-        print(doc)
-        pass
+        #url
+        self.__video_detail["url"] = doc('div>h3>a').attr("href")
+        #image
+        self.__video_detail["img"] = doc('img').filter('#video_jacket_img').attr("src")
+        #name 片名
+        print(doc('div>h3>a').text())
+        self.__video_detail["name"] = doc('div>h3>a').text()
+        #ID 识别码
+        doc2 = Pq(doc('div').filter("#video_id"))
+        self.__video_detail["id"] = doc2("td").filter(".text").text()
+        #maker 制作商
+        doc2 = Pq(doc('div').filter("#video_maker"))
+        self.__video_detail["maker"] = doc2("span").filter(".maker").text()
+        #tag
+        doc2 = Pq(doc('div').filter("#video_genres"))
+        for tag in doc2("a[rel='category tag']").text().split(" "):
+            if tag is not None:
+                self.__video_tag = {
+                    'video_id': self.__video_detail["id"],
+                    'tag': tag
+                }
+                self.__tags.append(self.__video_tag)
+            else:
+                continue
+        # cast #演员
+        doc2 = Pq(doc('div').filter("#video_cast"))
+        for cast in doc2("a[rel='tag']").text().split(" "):
+            if cast is not None:
+                self.__video_cast = {'video_id': self.__video_detail["id"], 'actor': cast}
+                print("video_cast is ", self.__video_cast)
+                self.__cast.append(self.__video_cast)
+                print(cast)
+            else:
+                continue
+        self._video_dao()
+
+    def _video_dao(self):
+        dao = DAO()
+        #表中是否已有记录
+        query_sql = "select * from av_info_main where video_id='{}' and maker = '{}'".format(self.__video_detail["id"],
+                                                                                             self.__video_detail["maker"])
+
+        if dao.execute_query(query_sql):
+            print("video{} is already exists ,so next".format(self.__cast[0]["video_id"]))
+            return
+        #数据插入操作
+        for video_cast1 in self.__cast:
+            #myset = video_cast1.split()
+            insert_sql = "INSERT INTO video_cast (video_id,actor)" \
+                         " VALUES ('{}','{}' )".format(video_cast1["video_id"], video_cast1["actor"])
+            dao.execute_dml(insert_sql)
+        for tag1 in self.__tags:
+            #myset = video_cast1.split()
+            # print(video_cast1["id"], video_cast1["name"], video_cast1["link"], video_cast1["img"])
+            insert_sql = "INSERT INTO av_tag (video_id,video_tag )" \
+                         " VALUES ('{}','{}' )".format(tag1["video_id"], tag1["tag"])
+            dao.execute_dml(insert_sql)
+        insert_sql = "INSERT INTO av_info_main (video_id,video_name,video_src,img,maker )" \
+                     " VALUES ('{}','{}','{}','{}','{}' )".format(self.__video_detail["id"], self.__video_detail["name"],
+                                                                  self.__video_detail["url"], self.__video_detail["img"],
+                                                                  self.__video_detail["maker"])
+        dao.execute_dml(insert_sql)
+
+if __name__ == '__main__':
+    v1 = VideoDetailCrawler("http://www.javlibrary.com/cn/?v=javlij3by4")
+    v1.craw()

video_list_crawler.py

@@ -4,7 +4,7 @@
 from pyquery import PyQuery as Pq
 
 from base_crawler import BaseCrawler
-
+from video_detail_crawler import  VideoDetailCrawler
 
 class VideoListCrawler(BaseCrawler):
     def __init__(self):
@@ -18,7 +18,7 @@ def _generate_seed_url(self):
         """
         generate all url to visit
         """
-
+        ##from page 1 to anypage which < 200
         for page_no in range(1, 2):
             self._seed_url.append(self._domain + self._info_uri + page_no.__str__())
 
@@ -30,4 +30,5 @@ def _extract_data(self, doc_str):
             video_id = video_id[4:]
             detail_url = self._domain + self._detail_uri + video_id
             self.detail_info_urls.append(detail_url)
-
+            crawler = VideoDetailCrawler(detail_url)
+            crawler.craw()