1. [代码]用python实现的抓取腾讯视频所有电影的爬虫
跳至
[1]
[全屏预览]
001 | # -*- coding: utf-8 -*- |
002 | # by awakenjoys. my site: www.dianying.at |
005 | from bs4 import BeautifulSoup |
010 | m_type = u'' #全局变量,电影类型 |
011 | m_site = u 'qq' #全局变量,电影网站 |
015 | req = urllib2.Request(url) |
016 | response = urllib2.urlopen(req) |
017 | html = response.read() |
023 | soup = BeautifulSoup(html) #过滤出分类内容 |
025 | #<ul class="clearfix _group" gname="mi_type" gtype="1"> |
026 | tags_all = soup.find_all( 'ul' , { 'class' : 'clearfix _group' , 'gname' : 'mi_type' }) |
027 | #print len(tags_all), tags_all |
028 | #print str(tags_all[1]).replace('\n', '') |
030 | #<a _hot="tag.sub" class="_gtag _hotkey" href="http://v.qq.com/list/1_0_-1_-1_1_0_0_20_0_-1_0.html" title="动作" tvalue="0">动作</a> |
031 | re_tags = r '<a _hot=\"tag\.sub\" class=\"_gtag _hotkey\" href=\"(.+?)\" title=\"(.+?)\" tvalue=\"(.+?)\">.+?</a>' |
032 | p = re. compile (re_tags, re.DOTALL) |
034 | tags = p.findall( str (tags_all[ 0 ])) |
039 | tag_url = tag[ 0 ].decode( 'utf-8' ) |
041 | m_type = tag[ 1 ].decode( 'utf-8' ) |
042 | tags_url[m_type] = tag_url |
049 | def get_pages(tag_url): |
050 | tag_html = gethtml(tag_url) |
051 | #div class="paginator |
052 | soup = BeautifulSoup(tag_html) #过滤出标记页面的html |
054 | #<div class="mod_pagenav" id="pager"> |
055 | div_page = soup.find_all( 'div' , { 'class' : 'mod_pagenav' , 'id' : 'pager' }) |
056 | #print div_page #len(div_page), div_page[0] |
058 | #<a class="c_txt6" href="http://v.qq.com/list/1_2_-1_-1_1_0_24_20_0_-1_0.html" title="25"><span>25</span></a> |
059 | re_pages = r '<a class=.+?><span>(.+?)</span></a>' |
060 | p = re. compile (re_pages, re.DOTALL) |
061 | pages = p.findall( str (div_page[ 0 ])) |
069 | def getmovielist(html): |
070 | soup = BeautifulSoup(html) |
072 | #<ul class="mod_list_pic_130"> |
073 | divs = soup.find_all( 'ul' , { 'class' : 'mod_list_pic_130' }) |
075 | for div_html in divs: |
076 | div_html = str (div_html).replace( '\n' , '') |
086 | #<h6 class="caption"> <a href="http://www.tudou.com/albumcover/Z7eF_40EL4I.html" target="_blank" title="徒步旅行队">徒步旅行队</a> </h6> <ul class="info"> <li class="desc">法国卖座喜剧片</li> <li class="cast"> </li> </ul> </div> <div class="ext ext_last"> <div class="ext_txt"> <h3 class="ext_title">徒步旅行队</h3> <div class="ext_info"> <span class="ext_area">地区: 法国</span> <span class="ext_cast">导演: </span> <span class="ext_date">年代: 2009</span> <span class="ext_type">类型: 喜剧</span> </div> <p class="ext_intro">理查德·达奇拥有一家小的旅游公司,主要经营法国游客到非洲大草原的旅游服务。六个法国游客决定参加理查德·达奇组织的到非洲的一...</p> |
088 | re_movie = r '<li><a class=\"mod_poster_130\" href=\"(.+?)\" target=\"_blank\" title=\"(.+?)\"><img.+?</li>' |
089 | p = re. compile (re_movie, re.DOTALL) |
090 | movies = p.findall(html) |
092 | conn = pymongo.Connection( 'localhost' , 27017 ) |
093 | movie_db = conn.dianying |
094 | playlinks = movie_db.playlinks |
099 | print "%s : %d" % ( "=" * 70 , NUM) |
101 | movie_title = movie[ 1 ], |
102 | movie_url = movie[ 0 ], |
107 | playlinks.insert(values) |
110 | print "%s : %d" % ( "=" * 70 , NUM) |
115 | def getmovieinfo(url): |
117 | soup = BeautifulSoup(html) |
119 | #pack pack_album album_cover |
120 | divs = soup.find_all( 'div' , { 'class' : 'pack pack_album album_cover' }) |
123 | #<a href="http://www.tudou.com/albumplay/9NyofXc_lHI/32JqhiKJykI.html" target="new" title="《血滴子》独家纪录片" wl="1"> </a> |
124 | re_info = r '<a href=\"(.+?)\" target=\"new\" title=\"(.+?)\" wl=\".+?\"> </a>' |
125 | p_info = re. compile (re_info, re.DOTALL) |
126 | m_info = p_info.findall( str (divs[ 0 ])) |
130 | print "Not find movie info" |
135 | def insertdb(movieinfo): |
137 | movie_db = conn.dianying_at |
138 | movies = movie_db.movies |
139 | movies.insert(movieinfo) |
141 | if __name__ = = "__main__" : |
144 | tags_url = "http://v.qq.com/list/1_-1_-1_-1_1_0_0_20_0_-1_0.html" |
146 | tags_html = gethtml(tags_url) |
148 | tag_urls = gettags(tags_html) |
152 | for url in tag_urls.items(): |
153 | print str (url[ 1 ]).encode( 'utf-8' ) #,url[0] |
154 | maxpage = int (get_pages( str (url[ 1 ]).encode( 'utf-8' ))) |
157 | for x in range ( 0 , maxpage): |
158 | #http://v.qq.com/list/1_0_-1_-1_1_0_0_20_0_-1_0.html |
159 | m_url = str (url[ 1 ]).replace( '0_20_0_-1_0.html' , '') |
160 | movie_url = "%s%d_20_0_-1_0.html" % (m_url, x) |
162 | movie_html = gethtml(movie_url.encode( 'utf-8' )) |
164 | getmovielist(movie_html) |
|