当前位置：首页>Python>正文

基于python的影評數據分析，python爬取《你好, 李煥英》豆瓣評論數據

# 導入工具包
import requests
from bs4 import BeautifulSoup
import time
import pandas as pd
import numpy as np# 請求頭
headers={'User-Agent':'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/88.0.4324.190 Safari/537.36'}
# =============================================================================
# 爬取一頁
# =============================================================================
# 爬取的網址
url='https://movie.douban.com/subject/34841067/comments?limit=20&status=P&sort=new_score'# 獲取信息
html = requests.get(url,headers=headers)
# 獲取內容
data = html.text
soup = BeautifulSoup(data,'lxml')# 信息
# 用戶
names = soup.select('#comments > div > div.comment > h3 > span.comment-info > a')
# 評級
pingjis = soup.select('#comments > div > div.comment > h3 > span.comment-info')
# 日期
riqis = soup.select('#comments > div > div.comment > h3 > span.comment-info > span.comment-time')
# 內容
neirongs = soup.select('#comments > div > div.comment > p > span')# 空list
lis=[]
for name,pingji,riqi,neirong in zip(names,pingjis,riqis,neirongs):pingji_re = pingji.find_all('span')lis.append([name.get_text(),pingji_re[1]['class'],pingji_re[1]['title'],riqi.get_text().strip(),neirong.get_text()])result1 = pd.DataFrame(lis,columns=['用戶','評級','等級','日期','內容'])# =============================================================================
# 爬取多頁
# =============================================================================
url = ['https://movie.douban.com/subject/34841067/comments?start={}&limit=20&status=P&sort=new_score'.format(i) for i in range(0,100,20)]lis2 = []for urli in url:# 獲取信息html = requests.get(urli,headers=headers)# 獲取內容data = html.textsoup = BeautifulSoup(data,'lxml')# 用戶names = soup.select('#comments > div > div.comment > h3 > span.comment-info > a')# 評級pingjis = soup.select('#comments > div > div.comment > h3 > span.comment-info')# 日期riqis = soup.select('#comments > div > div.comment > h3 > span.comment-info > span.comment-time')# 內容neirongs = soup.select('#comments > div > div.comment > p > span')for name,pingji,riqi,neirong in zip(names,pingjis,riqis,neirongs):pingji_re = pingji.find_all('span')lis2.append([name.get_text(),pingji_re[1]['class'],pingji_re[1]['title'],riqi.get_text().strip(),neirong.get_text()])print('完成:',urli)time.sleep(np.random.randint(5,10))result2 = pd.DataFrame(lis2,columns=['用戶','評級','等級','日期','內容'])