Python 代码优化：使用线程池加速 Google 搜索爬取

import\x20requests\nimport\x20time\nimport\x20os\nimport\x20urllib3\nimport\x20sys\nimport\x20random\nfrom\x20bs4\x20import\x20BeautifulSoup\nfrom\x20concurrent.futures\x20import\x20ThreadPoolExecutor\nfrom\x20threading\x20import\x20Lock\nfrom\x20colorama\x20import\x20Fore, init\n\nnow_time\x20=\x20time.strftime('%Y-%m-%d\x20%H-%M')\n\n\n#\x20读取Dorks\ndef\x20work(dorks):\n\x20with\x20open(dorks, mode='r', encoding='utf-8')\x20as\x20file:\n\x20\x20read_content\x20=\x20file.readlines()\n\x20\x20#\x20将内容加入列表\n\x20\x20content\x20=\x20[result.strip()\x20for\x20result\x20in\x20read_content]\n\x20\x20#\x20返回数量丢给任务池\n\x20\x20return\x20len(read_content), content\n\n\n#\x20Google搜索\ndef\x20google_serach(query, locks, filename):\n\x20try:\n\x20\x20#\x20关闭HTTPS报错信息\n\x20\x20urllib3.disable_warnings()\n\x20\x20filename\x20=\x20os.path.join(os.getcwd(), f'{filename}.txt')\n\x20\x20domains\x20=\x20['fr','it','ca','co.uk','ru','co,jp','co.kr','com.au','co.in','com.br','com.ar','co.za','co.nz','es','se','nl','ch','at','dk','be','pl','fi','ie','pt','gr', 'tw', 'com', 'uk', 'de', 'br', 'ca', 'kr', 'mx', 'au', 'za']\n\x20\x20random_domain\x20=\x20random.choice(domains)\n\x20\x20url\x20=\x20f'https://www.google.{random_domain}/search?q={query}&num=100'\n\x20\x20#\x20请求头\n\x20\x20headers\x20=\x20{\n\x20\x20\x20\x20'user-agent':\x20'Mozilla/5.0\x20(Windows\x20NT\x2010.0; Win64; x64)\x20AppleWebKit/537.36\x20(KHTML, like Gecko)\x20Chrome/111.0.0.0\x20Safari/537.36',\n\x20\x20\x20\x20'accept-language':\x20'zh-CN,zh;q=0.9',\n\x20\x20\x20\x20'accept':\x20'text/html,application/xhtml+xml,application/xml;q=0.9,image/avif,image/webp,image/apng,/;q=0.8,application/signed-exchange;v=b3;q=0.7',\n\x20\x20\x20\x20'referer':\x20'https://www.google.com/',\n\x20\x20\x20\x20'origin':\x20'https://www.google.com',\n\x20\x20\x20\x20'Sec-Fetch-Site':\x20'same-origin',\n\x20\x20\x20\x20'Sec-Fetch-Mode':\x20'navigate',\n\x20\x20\x20\x20'Sec-Fetch-User':\x20'?1',\n\x20\x20\x20\x20'Sec-Fetch-Dest':\x20'document'\n\x20\x20}\n\x20\x20#\x20代理\n\x20\x20proxies\x20=\x20{'http':\x20'http://127.0.0.1:7890', 'https':\x20'http://127.0.0.1:7890'}\n\x20\x20response\x20=\x20requests.get(url=url, headers=headers, proxies=proxies, verify=False, timeout=5)\n\x20\x20soup\x20=\x20BeautifulSoup(response.content, 'html.parser')\n\x20\x20#\x20查找全部div标签\n\x20\x20find_div\x20=\x20soup.find_all('div', {'class':\x20'yuRUbf'})\n\x20\x20#\x20开启线程锁\n\x20\x20locks.acquire()\n\x20\x20#\x20加入列表\n\x20\x20get_url\x20=\x20[url.findNext('a')['href']\x20+\x20'\n' for url in find_div if 'google.com' not in url.findNext('a')['href']]\n\x20\x20global url_num, dork_finish_num\n\x20\x20url_num\x20+=\x20len(get_url)\n\x20\x20dork_finish_num\x20+=\x201\n\x20\x20print(Fore.GREEN\x20+\x20f'\r{now_time}[INFO]{ '-' * 10}>get Urlnumber：{url_num} Dorsk number：{dork_finish_num} / {dork_total_num}', end='' + Fore.RESET)\n\x20\x20#\x20写入文件\n\x20\x20write_info(filename, get_url)\n\x20\x20#\x20释放线程锁\n\x20\x20locks.release()\n\n\x20except TimeoutError:\n\x20\x20pass\n\n\n#\x20写入文件函数\ndef\x20write_info(filename, get_url):\n\n\x20with\x20open(filename, mode='a+', encoding='utf-8')\x20as\x20file:\n\n\x20\x20file.writelines(get_url)\n\n\nif name == 'main':\n\x20while True:\n\x20\x20try:\n\x20\x20\x20init() # 初始化颜色模块\n\x20\x20\x20dorks_file\x20=\x20input(Fore.YELLOW\x20+\x20f'\n{now_time}[INFO]{ '-' * 10}>input file:' + Fore.RESET)\n\x20\x20\x20print('')\n\x20\x20\x20filename\x20=\x20input(Fore.YELLOW\x20+\x20f'\n{now_time}[INFO]{ '-' * 10}>output file:' + Fore.RESET)\n\x20\x20\x20#\x20接受work函数返回的元组\n\x20\x20\x20dork_total_num, query_list\x20=\x20work(dorks_file)\n\x20\x20\x20#\x20定义全局变量完成数量/URL数量\n\x20\x20\x20dork_finish_num\x20=\x20url_num\x20=\x200\n\n\x20\x20\x20#\x20定义线程池数量\n\x20\x20\x20executor\x20=\x20ThreadPoolExecutor(max_workers=20)\n\n\x20\x20\x20#\x20定义全局锁\n\x20\x20\x20threads_lock\x20=\x20Lock()\n\n\x20\x20\x20#\x20分配进程池任务\n\x20\x20\x20for dork_list in query_list:\n\x20\x20\x20\x20executor.submit(google_serach, dork_list, threads_lock, filename)\n\n\x20\x20\x20executor.shutdown()\n\x20\x20\x20if len(sys.argv) == 1:\n\x20\x20\x20\x20pass\n\x20\x20\x20input(Fore.YELLOW\x20+\x20f'\n\n{now_time}[INFO]{'-' * 10}>final huiche' + Fore.RESET)\n\x20\x20\x20break\n\x20\x20#\x20文件为空\n\x20\x20except FileNotFoundError:\n\x20\x20\x20print(Fore.RED\x20+\x20f'{now_time}[Error]{'-' * 10}>fiel not find' + Fore.RESET)\n\x20\x20#\x20中断异常\n\x20\x20except KeyboardInterrupt:\n\x20\x20\x20sys.exit(1)