Python爬虫:简易的爬取斗鱼弹幕
首先看一下协议的内容
斗鱼后台协议头设计:
请求一共分为三个部分:长度,头部,数据部
分别按照文档的要求构造就行,
需要注意的是,获取和返回的类型是都是 Bytes
def send_request_msg(msgstr):
msg = msgstr.encode('utf-8') # 协议规定所有协议内容均为 UTF-8 编码
data_lenth = len(msg) + 8
# data_lenth表示整个协议头的长度(消息长度),包括数据部分和头部,len(msg)就是数据部分,8就是头部的长度
code = 689
# 根据协议消息类型字段用689
msghead = int.to_bytes(data_lenth, 4, 'little') + int.to_bytes(data_lenth, 4, 'little') + int.to_bytes(code, 4, 'little')
# msghead是按照斗鱼第三方协议构造的协议头,前2段表示的是消息长度,最后一个是消息类型
client.send(msghead) # 发送协议头
client.send(msg) # 发送消息请求
然后是获取弹幕:
也是按照文档要求写就成。首先发送登录请求,接着每隔固定时间发送心跳请求防止断线
def get_bulletscreen(roomid):
id_msg_list = []
denglu_msg = '[email protected]=loginreq/[email protected]={}/\0'.format(roomid) # 登录请求消息,最后面的'\0',是协议规定在数据部分结尾必须是'\0'
send_request_msg(denglu_msg)
join_room_msg = '[email protected]=joingroup/[email protected]={}/[email protected]=-9999/\0'.format(roomid) # 加入房间分组消息
send_request_msg(join_room_msg)
while True:
data = client.recv(1024)
# 这个data就是服务器向客户端发送的消息
bulletscreen_username = re.findall(user_id, data)
bulletscreen_content = re.findall(bulletscreen, data)
# print(data)
if not data:
break
else:
for i in range(0, len(bulletscreen_username)):
try:
print('[{}]:{}'.format(bulletscreen_username[i].decode('utf-8'), bulletscreen_content[i].decode('utf-8')))
# 返回的数据是bytes型,所以要用decode方法来解码
id_msg_list.append(bulletscreen_username[i].decode('utf-8'))
id_msg_list.append(bulletscreen_content[i].decode('utf-8'))
except:
continue
# 维持与后台的心跳
def keeplive():
while True:
live_msg = '[email protected]=keeplive/[email protected]=' + str(int(time.time())) + '/\0'
send_request_msg(live_msg)
time.sleep(15)
完整代码
import re
import socket
import signal
import multiprocessing
import time
client = socket.socket(socket.AF_INET, socket.SOCK_STREAM)
port = 8602 # 端口8601、8602、12601、12602这几个端口号都是
host = socket.gethostbyname('openbarrage.douyutv.com')
client.connect((host, port))
bulletscreen = re.compile(b'[email protected]=(.+?)/')
user_id = re.compile(b'[email protected]=(.+?)/')
def send_request_msg(msgstr):
msg = msgstr.encode('utf-8') # 协议规定所有协议内容均为 UTF-8 编码
data_lenth = len(msg) + 8
# data_lenth表示整个协议头的长度(消息长度),包括数据部分和头部,len(msg)就是数据部分,8就是头部的长度
code = 689
# 根据协议消息类型字段用689
msghead = int.to_bytes(data_lenth, 4, 'little') + int.to_bytes(data_lenth, 4, 'little') + int.to_bytes(code, 4, 'little')
# msghead是按照斗鱼第三方协议构造的协议头,前2段表示的是消息长度,最后一个是消息类型
client.send(msghead) # 发送协议头
client.send(msg) # 发送消息请求
def get_bulletscreen(roomid):
id_msg_list = []
denglu_msg = '[email protected]=loginreq/[email protected]={}/\0'.format(roomid) # 登录请求消息,最后面的'\0',是协议规定在数据部分结尾必须是'\0'
send_request_msg(denglu_msg)
join_room_msg = '[email protected]=joingroup/[email protected]={}/[email protected]=-9999/\0'.format(roomid) # 加入房间分组消息
send_request_msg(join_room_msg)
while True:
data = client.recv(1024)
# 这个data就是服务器向客户端发送的消息
# 具体的信息可以看斗鱼弹幕第三方接入协议
bulletscreen_username = re.findall(user_id, data)
bulletscreen_content = re.findall(bulletscreen, data)
# print(data)
if not data:
break
else:
for i in range(0, len(bulletscreen_username)):
try:
print('[{}]:{}'.format(bulletscreen_username[i].decode('utf-8'), bulletscreen_content[i].decode('utf-8')))
# 返回的数据是bytes型,所以要用decode方法来解码
id_msg_list.append(bulletscreen_username[i].decode('utf-8'))
id_msg_list.append(bulletscreen_content[i].decode('utf-8'))
except:
continue
def keeplive():
# 维持与后台的心跳
# 关于心跳消息,协议中有详细的解释
while True:
live_msg = '[email protected]=keeplive/[email protected]=' + str(int(time.time())) + '/\0'
send_request_msg(live_msg)
time.sleep(15)
def logout():
out_msg = '[email protected]=logout/'
send_request_msg(out_msg)
print('已退出服务器!')
def signal_handler(signal, frame):
# 捕捉ctrl + c的信号,即signal.SIGINT
p1.terminate() # 结束进程
p2.terminate() # 结束进程
logout()
if __name__ == '__main__':
roomid = 9999 # 房间号,主播开播才能获取到信息
signal.signal(signal.SIGINT, signal_handler)
p1 = multiprocessing.Process(target=get_bulletscreen, args=(roomid,))
p2 = multiprocessing.Process(target=keeplive)
p1.start()
p2.start()
运行结果
有些用户名没有获取到是因为网页版搞得什么角色弹幕,是一个图片