|
import urllib.request from bs4 import BeautifulSoup import time def main(): url = 'http://www.shicimingju.com/book/' #获得请求对象 request = get_request(url) #获得响应参数 html = get_response(request) #获取数据 soup_html(html) def get_request(url): #获得请求对象 headers={ 'User-Agent': 'Mozilla/5.0 (Windows NT 6.1; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/67.0.3396.99 Safari/537.36' } request = urllib.request.Request(url=url,headers=headers) return request def get_response(req): response = urllib.request.urlopen(req) html = response.read().decode('utf8') return html #获得第一层数据 def soup_html(html): #实例化soup soup = BeautifulSoup(html,'lxml') ret = soup.select('.bookmark-list ul li h2 a') for i in ret: title = i.text href = 'http://www.shicimingju.com' + i['href'] # print(href) #调用生成请求对象的方法 request = get_request(href) #调用生成响应对象的方法 text = get_response(request) filename = title + '.txt' fp = open(filename,'a',encoding='utf8') # print(text) # 调用第二层的生成的数据 datas = get_txt(href) for i in datas: title = i[0] print("正在下载%s" %title) for i in i[1]: res = i.get_text() fp.write(title+'\n'+res) time.sleep(2) print("下载结束%s"%title) # print("datas:\n",datas) # # #获得第二层数据 def get_txt(href): # #调用生成请求对象的方法 request = get_request(href) # #调用生成响应对象的方法 text = get_response(request) # print(text) #实例化一个soup soup = BeautifulSoup(text,'lxml') ret = soup.select('.book-mulu ul li a') # print(ret) print(1) d1 = [] for i in ret: title = i.text href = 'http://www.shicimingju.com' + i['href'] # print(title,href) res = get_data(href) d1.append(res) return d1 #获得第三层数据 def get_data(href): #调用生成对象的方法 request = get_request(href) #调用生成响应对象的方法 response = get_response(request) #实例化一个soup soup = BeautifulSoup(response,'lxml') ret = soup.select('.www-shadow-card h1')# title = ret[0].text#获得每个章节的章回名 if soup.select('.chapter_content p'): res = soup.select('.chapter_content p') return title, res elif soup.select('.chapter_content'): res = soup.select('.chapter_content') return title, res #循环第三层数据 if __name__ == '__main__': main()
|
|
|
