在上一篇文章里介绍了利用find_all的方法来对网络上电影的排名，今天我们介绍一个利用select函数的方法。
方法二：利用select函数
#x1=soup.find_all("li")
x1 = soup.select("ol li") #子目录标签，各级标签之间利用空格进行分割
1.5影片短评的爬取
#循环获得短影评，电影编号 32659890
import requests as rt
from bs4 import BeautifulSoup as bs
headers = {"User-Agent": 'Mozilla/5.0 (Windows NT 10.0; Win64; x64) '
'AppleWebKit/537.36 (KHTML, like Gecko) Chrome/78.0.3904.108 Safari/537.36'
}
#循环获得短影评
k=range(0,1200,20)
my_txt=""
for i in k:
#my_url=my_list1[0]+r"comments?start="+str(i)+r"&limit=20&sort=new_score&status=P"
my_url =r"https://movie.douban.com/subject/32659890/"\ #电影编号32659890
+ r"comments?start=" + str(i) \
+ r"&limit=20&sort=new_score&status=P"
#print(my_url)
my_data=rt.get(my_url, headers=headers, timeout=30)
my_data.encoding="utf-8" #解决中文乱码问题
soup=bs(my_data.text,"html.parser")
#print(soup)
x1=soup.find_all("span",class_="short")
for i in x1:
#print(i.text)
my_txt=my_txt+i.text
#词频统计
import jieba
x3=jieba.lcut(my_txt)#分词后的文章
x4=dict()
#屏蔽词
y1=["\r\n","一部"]
for i in x3:
if i not in y1 and len(i) > 1:
x4[i]=x4.get(i,1)+1
#排序
res = sorted(x4.items(),key=lambda d:d[1],reverse=True)
#print(type(res),len(res))
for i in range(0,10):#词频的前10名
print(res[i])
#词云
from wordcloud import WordCloud
#图形
import cv2 #pip install opencv-python
mask =cv2.imread("000.jpg") #读取背景图片
my_txt2=" ".join(x3)
excludes=["\r\n","一部","电影","片子","没有","这部","一个"]
wordcloud = WordCloud(background_color="white",
width=800,
height=600,
font_path="msyh.ttc",
max_words=200,
max_font_size=200,
stopwords = excludes,
mask=mask
).generate(my_txt2)
wordcloud.to_file('111.jpg')
#显示图片
from PIL import Image
img=Image.open('111.jpg')
img.show()
1.6 爬取时光网排名
import requests
from bs4 import BeautifulSoup as bs
x2=range(1,11)
for my_n in x2:
??? if my_n==1:
??????? my_url=r"http://www.mtime.com/top/movie/top100/"
??? else:
??????? my_url=r"http://www.mtime.com/top/movie/top100/index-"+str(my_n)+r".html"
??? print(my_url)
??? my_data=requests.get(my_url)
??? my_data.encoding="utf-8" #解决中文乱码问题
??? soup=bs(my_data.text,"html.parser")
??? x1=soup.find_all('li')
??? for i in x1:
??????? if i.find("div",class_="number")!=None:
??????????? print(i.find("em").text,end=" ")
??????????? try:
??????????????? print(i.find("a",class_="c_fff").text,end=" ")
??????????? except:
??????????????? print(i.find("a",class_="c_blue").text,end=" ")
??????????? print(i.find("span",class_="total").text,end="")
??????????? print(i.find("span", class_="total2").text)
