Python实战-采集斗鱼舞蹈区美女~(批量采集)

python芊 2023-02-06 发布于湖南

展开全文

前言

大家早好、午好、晚好吖 ❤ ~

开发环境:

python 3.8: 解释器
pycharm: 代码编辑器
requests: 发送请求
execjs: 执行js代码 pyexecjs
nodejs+crypto-js模块
pycharm--nodejs插件

基本流程

一. 数据来源分析

抓包数据包

m3u8: 视频流 m3u8文本类型文件将 mp4 切分为n个片段(.ts片段视频) 每一个片段为10秒钟

二. 代码实现(爬虫实现的基本流程)

发送请求
获取数据
解析数据
保存数据
批量采集视频

代码展示

这里网址我就屏蔽了，大家根据我上文添加完整

导入模块

import requestsimport reimport timeimport execjsfrom tqdm import tqdm

请求头(伪装)

vid = 'kDe0W2m4JO2MA4Bz'headers = {    'cookie': 'dy_did=6a87143ffbf77633191f343700031601; Hm_lvt_e99aee90ec1b2106afe7ec3b199020a7=1675325976; dy_did=6a87143ffbf77633191f343700031601',    'origin': 'https://v.****.com',    'referer': f'https://v.****.com/show/{vid}',    'user-agent': 'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/109.0.0.0 Safari/537.36'}

"""

获取point_id

:param vid: 作品id

:return: point_id

"""


    point_url = f"https://v.****.com/wgapi/vod/front/video/secondary/info?hid={vid}"
    point_id = requests.get(point_url, headers=headers).json()['data']['vid']    return point_id

"""

获取sign

:param vid: 作品id

:param point_id: point_id

:param did: did

:param tt: 时间戳

:return: sign值

"""

def get_sign(vid, point_id, did, tt):
    main_url = f'https://v.****.com/show/{vid}'
    html_data = requests.get(main_url, headers=headers).text
    title = re.findall('<title>(.*?)</title>', html_data)[0]
    f = "const CryptoJS = require('crypto-js');" + re.findall('<script> (var v.*?)</script>', html_data)[0]
    ctx = execjs.compile(f)
    result = ctx.call('ub98484234', point_id, did, tt)
    sign = result.split('sign=')[-1]    return [sign, title]

"""

获取m3u8链接

:param v: 固定参数

:param did: 固定参数

:param tt: 时间戳

:param sign: 加密参数

:param vid: 视频id

:return: m3u8链接

"""

def get_streamUrl(v, did, tt, sign, vid):

    data = {        'v': v,        'did': did,        'tt': str(tt),        'sign': sign,        'vid': vid
    }

发送请求

    url = 'https://v.****.com/api/stream/getStreamUrl'
    response = requests.post(url=url, headers=headers, data=data)

获取数据

    m3u8_url = response.json()['data']['thumb_video']['high']['url']    return m3u8_url

"""

下载视频

:param m3u8_url: 链接

:param title: 标题

"""

def download_m3u8(m3u8_url, title):
    sub_ts = m3u8_url.split('playlist')[0]
    m3u8_text = requests.get(m3u8_url).text
    ts_list = re.sub('#E.*', '', m3u8_text).split()    for ts in tqdm(ts_list):
        ts_url = sub_ts+ts
        ts_data = requests.get(ts_url).content        with open(f'{title}.mp4', mode='ab') as f:
            f.write(ts_data)


did = "10000000000000000000000000001501"tt = int(time.time())
v = '220320230203'point_id = get_point_id(vid)
info = get_sign(vid, point_id, did, tt)
sign = info[0]
title = info[1]
m3u8_url = get_streamUrl(v, did, tt, sign, vid)
download_m3u8(m3u8_url, title)