import time import re from selenium import webdriver from selenium.webdriver.chrome.options import Options from bs4 import BeautifulSoup

Using Chrome browser driver and setting it to headless mode

chrome_options = Options() chrome_options.add_argument("--headless") driver = webdriver.Chrome(options=chrome_options)

Sending request and retrieving webpage content

url = "https://search.jd.com/Search?keyword=%E4%BA%BA%E5%B7%A5%E6%99%BA%E8%83%BD%E5%9B%BE%E4%B9%A6&enc=utf-8&suggest=1.his.0.0&wq=&pvid=bc47d80392dd4b19980a4ca7ea7de082" driver.get(url)

Delay to allow the page to load completely

time.sleep(5)

Retrieving html content after the page has fully loaded

html_content = driver.page_source

Using BeautifulSoup to parse the html content

soup = BeautifulSoup(html_content, "html.parser")

Finding all divs with class "gl-i-wrap" containing product information

div_list = soup.find_all("div", class_="gl-i-wrap")

Extracting text information from each div

for div in div_list: name = div.find("div", class_="p-name").get_text() price = div.find("div", class_="p-price").get_text() commit = div.find("div", class_="p-commit").get_text() commit = commit.replace('条评价', '').replace('+', '') if '万' in commit: commit = float(commit.replace('万', '')) * 10000

#simulate click on the name, collect the information in the new page
link = div.find("div", class_="p-name").find("a").get("href")
if "http" not in link:
    link = "https:" + link

#open new tab
driver.execute_script(f'''window.open("{link}","_blank");''')
#switch to the new tab
windows = driver.window_handles
driver.switch_to.window(windows[-1])
time.sleep(5)
soup_new = BeautifulSoup(driver.page_source, "html.parser")
submit_button = driver.find_element_by_xpath("//*[@id='detail']/div[1]/ul/li[5]")
submit_button.click()
time.sleep(1)
publisher = soup_new.find("li", title=True, clstag="shangpin|keycount|product|chubanshe_3")["title"]\
    if soup_new.find("li", title=True, clstag="shangpin|keycount|product|chubanshe_3") is not None else '未找到'
publish_date = soup_new.find("li", text = re.compile(r"出版时间：")).get_text().replace('出版时间：', '') \
    if soup_new.find("li", text = re.compile(r"出版时间：")) is not None else '未找到'

driver.close()
driver.switch_to.window(windows[0])

print("Name:", name)
print("Price:", price)
print("Reviews:", commit)
print("Publishers:", publisher)
print("Publish Date:", publish_date)

#simulate click on the name, collect the information in the new page
link = div.find("div", class_="p-name").find("a").get("href")
if "http" not in link:
    link = "https:" + link

#open new tab
driver.execute_script(f'''window.open("{link}","_blank");''')
#switch to the new tab
windows = driver.window_handles
driver.switch_to.window(windows[-1])
time.sleep(5)
soup_new = BeautifulSoup(driver.page_source, "html.parser")
submit_button = driver.find_element_by_xpath("//*[@id='detail']/div[1]/ul/li[5]")
submit_button.click()
time.sleep(1)
# 爬取并输出评价信息(好评中评差评数目)
good_comments = driver.find_elements_by_xpath("//*[@id='comment']/div[2]/div[2]/div[1]/ul/li[5]/a/em")
for comment in good_comments:
    comment_text = comment.text.strip("()+")
    if "万" in comment_text:
        comment_text = str(int(float(comment_text.strip("万")) * 10000))
    print("好评:" + comment_text)
medium_comments = driver.find_elements_by_xpath("//*[@id='comment']/div[2]/div[2]/div[1]/ul/li[6]/a/em")
for comment in medium_comments:
    comment_text = comment.text.strip("()+")
    if "万" in comment_text:
        comment_text = str(int(float(comment_text.strip("万")) * 10000))
    print("中评:" + comment_text)

bad_comments = driver.find_elements_by_xpath("//*[@id='comment']/div[2]/div[2]/div[1]/ul/li[7]/a/em")
for comment in bad_comments:
    comment_text = comment.text.strip("()+")
    if "万" in comment_text:
        comment_text = str(int(float(comment_text.strip("万")) * 10000))
    print("差评:" + comment_text)

driver.close()
driver.switch_to.window(windows[0])

Closing the browser driver

driver.quit(