爬虫基本流程 re正则表达式 (内置模块) requests >>> pip install requests 在CMD 命令符 win + R json数据解析方法 视频数据保存
找数据对应的地址 使用python代码发送请求 数据筛选 数据保存
import requests # 数据请求 第三方模块 pip install requestsimport re # 正则表达式模块 内置模块from selenium import webdriver # 测试模拟 模拟人去操作浏览器 pip install seleniumimport pprint # 格式化输出模块import time # 时间模块# 需要谷歌/火狐驱动 python的环境安装在哪 就放那driver = webdriver.Chrome() # 把驱动直接放在python安装的路径里面 实例化一个浏览器对象driver.get('https://v.huya.com/g/all?set_id=31&order=hot&page=1')def get_video_content(): # time.sleep(2) driver.refresh()
driver.implicitly_wait(10) # 隐式等待 等待数据加载 加载完成之后才继续运行后面的内容
# time 延时有点区别 死等
lis = driver.find_elements_by_css_selector('.vhy-video-list li') for li in lis:
video_url = li.find_element_by_css_selector('.video-wrap').get_attribute('href') print(video_url)
video_id = re.findall('https://v\.huya\.com/play/(.*?)\.html', video_url)[0]
headers = { # 'Cookie': 'SoundValue=0.50; isInLiveRoom=; udb_guiddata=f88bdbcfecb444cbaebfd3430e0c220c; udb_deviceid=w_491619378696527872; udb_anouid=1462126356760; Hm_lvt_51700b6c722f5bb4cf39906a596ea41f=1631947193; __yasmid=0.5061768216828664; __yamid_tt1=0.5061768216828664; __yamid_new=C986054DD8700001912A6690BBA03A50; _yasids=__rootsid%3DC986054DD8900001E8251C028DB01560; Hm_lvt_9fb71726843792b1cba806176cecfe38=1631947194; udb_passdata=3; hiido_ui=0.8655862701494763; Hm_lpvt_51700b6c722f5bb4cf39906a596ea41f=1631948597; Hm_lpvt_9fb71726843792b1cba806176cecfe38=1631948597; rep_cnt=96',
# 'Host': 'v.huya.com',
# 'Pragma': 'no-cache',
# 'Upgrade-Insecure-Requests': '1',
'User-Agent': 'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/93.0.4577.82 Safari/537.36',
}
index_url = f'https://liveapi.huya.com/moment/getMomentContent?videoId={video_id}&uid=&_=1631947292202'
json_data = requests.get(url=index_url, headers=headers).json() # json字典数据 可以直接根据键值对 提取数据内容 冒号左边 提取冒号右边的
play_url = json_data['data']['moment']['videoInfo']['definitions'][0]['url']
title = json_data['data']['moment']['videoInfo']['videoTitle'] # video_content = requests.get(url=play_url, headers=headers).content # 获取二进制数据内容
# with open('video\\' + title + '.mp4', mode='wb') as f:
# f.write(video_content)
print(title, play_url)for page in range(1, 3): print(f'正在爬取第{page}页数据内容')
get_video_content()
driver.find_element_by_css_selector('.next').click()
time.sleep(1) |
