豆瓣电影Top250爬取及数据分析 - Python代码示例

import time//nimport random//nimport requests//nfrom bs4 import BeautifulSoup//nimport re//nimport pandas as pd//nimport openpyxl//n//nurl = 'https://movie.douban.com/top250'//nheaders = {//n 'User-Agent':'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/114.0.0.0 Safari/537.36 Edg/114.0.1823.43',//n}//nr = requests.get(url, headers=headers) # 获取网页//n# print(html)//n# print(soup)//ndata = []//n//ndef listpage_url(page):#获取列表页网址//n listpage_url = 'https://movie.douban.com/top250?start={}&filter='.format(page * 25)//n return listpage_url//n//ndef infomation(url)://n html = r.text//n soup = BeautifulSoup(html, 'lxml') # 转为html//n body = soup.body//n datas = body.find('div', attrs={'id': 'content'})//n ol = datas.find('ol')//n movie_list = ol.find_all('li')//n infos = soup.find(name='ol', class_=/'grid_view/').find_all(name='li') # 找到所有影片总信息//n for info in infos://n ranks = info.find(name='em', class_='').get_text() # 获取排名//n print(ranks)//n poster = info.find(name='div', class_='pic').find(name='img')['src'] # 获取海报//n print(poster)//n title = info.find(name='div', class_='info').find(name='span', class_='title').get_text() # 获取片名//n print(title)//n rating = info.find(name='div', class_='star').find(name='span', class_='rating_num').get_text() # 获取评分//n print(rating)//n total = info.find(name='div', class_='info').find(name='div', class_='bd').find('p').get_text() # 获取整合信息//n # print(total)//n director = re.search(r'导演:/s(.?)/s', total).group(1) # 获取导演//n print(director)//n actors = re.search(r'主演:/s(.)[.../]', total).group(1) # 获取主演//n print(actors)//n time = re.search(r'(/d{4})', total, re.S).group(1) # 获取时间//n print(time)//n local = re.search(r'/d*/s//(.)//', total, re.S).group(1) # 获取地区//n print(local)//n type = re.search(r'//(.)///s(.*)/s', total, re.S).group(2) # 获取类型//n print(type)//n if int(ranks) <= 10://n comment = get_5comment(url) # 前十评论//n elif info.find(name='div', class_='info').find(name='p', class_='quote') and int(ranks) > 10://n comment = info.find(name='div', class_='info').find(name='p', class_='quote').find('span').get_text()#第一条//n elif info.find(name='div', class_='info').find(name='p', class_='quote') is None and int(ranks) > 10://n comment = '暂无评论'//n//n a_data = {#创建字典//n '排名': ranks,//n '海报': poster,//n '片名': title,//n '评分': rating,//n '导演': director,//n '主演': actors,//n '上映时间': time,//n '上映地点': local,//n '类型': type,//n '热门短评': comment//n }//n data.append(a_data)//n//n//ndef get_5comment(detail_url):#5条热门短评//n html = requests.get(url,headers =headers).text//n soup = BeautifulSoup(html, 'html.parser')//n #print(soup)//n comments = soup.find(name='div', id=/'hot-comments/', class_=/'tab/').find_all('span', class_='short')//n comment_list = []//n for i, com in enumerate(comments)://n comment = str(i+1)+' .'+com.get_text()//n comment_list.append(comment)//n return comment_list//n