ExamA-人工智能2104-2021010001-张三

import time import json from selenium import webdriver from bs4 import BeautifulSoup import requests

创建浏览器对象

browser = webdriver.Chrome()

访问url地址

browser.get('https://movie.douban.com/subject/25868125/')

定位元素并点击

btn = browser.find_element_by_css_selector('#comments-section > div:nth-child(1) > h2 > span > a') btn.click()

请求头提取

headers = { 'User-Agent': 'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/93.0.4577.63 Safari/537.36', 'Referer': 'https://movie.douban.com/subject/25868125/comments?status=P' }

初始页面url确定

url = 'https://movie.douban.com/subject/25868125/comments?start=0&limit=20&status=P&sort=new_score'

遍历页面循环

data = [] for i in range(3): # 发送请求并获得网页数据 response = requests.get(url, headers=headers)

# 解析网页结构
soup = BeautifulSoup(response.text, 'html.parser')

# 定位评论人名称、评论时间以及评论
comments = soup.find_all('div', class_='comment-item')
for comment in comments:
    name = comment.find('a', class_='').text.strip()
    time = comment.find('span', class_='comment-time').text.strip()
    content = comment.find('span', class_='short').text.strip()
    data.append({'name': name, 'time': time, 'content': content})

# 页面翻页规律设置
url = 'https://movie.douban.com/subject/25868125/comments?start={}&limit=20&status=P&sort=new_score'.format(i*20)

# 间隔3秒爬取下一页
time.sleep(3)

文件数据写入

with open('comments.json', 'w', encoding='utf-8') as f: # 文件定义json格式 json.dump(data, f, ensure_ascii=False, indent=4)

关闭浏览器

browser.quit()