人民日报新闻文章抓取工具 - Python代码实现

import requests
import bs4
import os
import time

def fetchUrl(url):
    headers = {
        'User-Agent': 'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/88.0.4324.96 Safari/537.36'
    }
    r = requests.get(url, headers=headers)
    r.raise_for_status()
    r.encoding = r.apparent_encoding
    return r.text

def NewsCategories():
    url = 'http://paper.people.com.cn/rmrb/html/2023-08/19/nbs.D110000renmrb_01.htm'
    html = fetchUrl(url)
    bsobj = bs4.BeautifulSoup(html, 'html.parser')
    categories = []
    temp = bsobj.find('div', attrs={'id': 'page'})
    if temp:
        categories = [a.text for a in temp.ul.find_all('a')]
    return categories

def getPageList(year, month, day, category):
    url = f'http://paper.people.com.cn/rmrb/html/{year}-{month}/{day}/nbs.D110000renmrb_01.htm'
    html = fetchUrl(url)
    bsobj = bs4.BeautifulSoup(html, 'html.parser')
    pageList = []
    temp = bsobj.find('div', attrs={'id': 'titleList'})
    if temp:
        pageList = temp.ul.find_all('li')
    return [page.a.get('href') for page in pageList if category in page.text]

def getContent(pageUrl):
    url = f'http://paper.people.com.cn/rmrb/html/{pageUrl}'
    html = fetchUrl(url)
    bsobj = bs4.BeautifulSoup(html, 'html.parser')
    content = bsobj.find('div', attrs={'class': 'text_c'}).get_text()
    return content.strip()

def saveFile(content, path, filename):
    save_path = os.path.join(path, filename)
    if not os.path.exists(path):
        os.makedirs(path)
    with open(save_path, 'w', encoding='utf-8') as f:
        f.write(content)

def calculateAccuracy(category, total_samples, category_count):
    accuracy = category_count / total_samples
    return accuracy * 100

def predictCategory(content):
    predicted_category = 'Example Category'
    return predicted_category

def downloadArticles(beginDate_str, endDate_str, category, save_path):
    category_count = 0
    total_samples = 0
    correct_predictions = 0

    beginDate = time.mktime(time.strptime(beginDate_str, '%Y%m%d'))
    endDate = time.mktime(time.strptime(endDate_str, '%Y%m%d'))

    date_diff = int((endDate - beginDate) / 86400) + 1

    for i in range(date_diff):
        current_date = time.strftime('%Y%m%d', time.localtime(beginDate + i * 86400))
        year = current_date[:4]
        month = current_date[4:6]
        day = current_date[6:]

        try:
            pageList = getPageList(year, month, day, category)

            for pageUrl in pageList:
                category_count += 1
                total_samples += 1

                try:
                    content = getContent(pageUrl)
                    prediction = predictCategory(content)

                    if prediction == category:
                        correct_predictions += 1

                    filename = f'{year}{month}{day}_{pageUrl}.txt'
                    saveFile(content, save_path, filename)
                    print(f'Successfully downloaded {filename}')
                except Exception as e:
                    print(f'Error occurred while downloading: {str(e)}')

        except requests.exceptions.HTTPError as e:
            print(f'Requested page not found: {str(e)}')

    accuracy = calculateAccuracy(category, total_samples, correct_predictions)

    print(f'Total samples: {total_samples}')
    print(f'Correct predictions: {correct_predictions}')
    print(f'Accuracy: {accuracy}%')

if __name__ == '__main__':
    beginDate_str = input('输入开始时间 (YYYYMMDD): ')
    endDate_str = input('输入结束时间 (YYYYMMDD): ')
    category = input('输入关键词: ')
    save_path = input('输入的路径: ')
    downloadArticles(beginDate_str, endDate_str, category, save_path)