Python Google Dork 爬虫工具 - 自动抓取搜索结果

import\x20requests\nimport\x20time\nimport\x20os\nimport\x20urllib3\nimport\x20sys\nimport\x20random\nfrom\x20bs4\x20import\x20BeautifulSoup\nfrom\x20concurrent.futures\x20import\x20ThreadPoolExecutor\nfrom\x20threading\x20import\x20Lock\nfrom\x20colorama\x20import\x20Fore,\x20init\n\nnow_time\x20=\x20time.strftime('%Y-%m-%d\x20%H-%M')\n\n#\x20关闭HTTPS报错信息\nurllib3.disable_warnings()\n\n#\x20读取Dorks\ndef\x20work(dorks):\n\x20\x20with\x20open(dorks,\x20mode='r',\x20encoding='utf-8')\x20as\x20file:\n\x20\x20\x20\x20read_content\x20=\x20file.readlines()\n\x20\x20\x20\x20#\x20将内容加入列表\n\x20\x20\x20\x20content\x20=\x20[result.strip()\x20for\x20result\x20in\x20read_content]\n\x20\x20\x20\x20#\x20返回数量丢给任务池\n\x20\x20\x20\x20return\x20len(read_content),\x20content\n\n\n#\x20Google搜索\ndef\x20google_serach(query,\x20locks,\x20filename):\n\x20\x20try:\n\x20\x20\x20\x20filename\x20=\x20os.path.join(os.getcwd(),\x20f'{filename}.txt')\n\x20\x20\x20\x20domains\x20=\x20['fr',\x20'it',\x20'ca',\x20'co.uk',\x20'ru',\x20'co,jp',\x20'co.kr',\x20'com.au',\x20'co.in',\x20'com.br',\x20'com.ar',\x20'co.za',\n\x20\x20\x20\x20\x20\x20\x20\x20'co.nz',\x20'es',\x20'se',\x20'nl',\x20'ch',\x20'at',\x20'dk',\x20'be',\x20'pl',\x20'fi',\x20'ie',\x20'pt',\x20'gr',\x20'tw',\x20'com',\x20'uk',\n\x20\x20\x20\x20\x20\x20\x20\x20'de',\x20'br',\x20'ca',\x20'kr',\x20'mx',\x20'au',\x20'za']\n\x20\x20\x20\x20random_domain\x20=\x20random.choice(domains)\n\x20\x20\x20\x20url\x20=\x20f'https://www.google.{random_domain}/search?q={query}&num=100'\n\x20\x20\x20\x20#\x20请求头\n\x20\x20\x20\x20headers\x20=\x20{\n\x20\x20\x20\x20\x20\x20'user-agent':\x20'Mozilla/5.0\x20(Windows\x20NT\x2010.0;\x20Win64;\x20x64)\x20AppleWebKit/537.36\x20(KHTML,\x20like\x20Gecko)\x20Chrome/111.0.0.0\x20Safari/537.36',\n\x20\x20\x20\x20\x20\x20'accept-language':\x20'zh-CN,zh;q=0.9',\n\x20\x20\x20\x20\x20\x20'accept':\x20'text/html,application/xhtml+xml,application/xml;q=0.9,image/avif,image/webp,image/apng,/;q=0.8,application/signed-exchange;v=b3;q=0.7',\n\x20\x20\x20\x20\x20\x20'referer':\x20'https://www.google.com/',\n\x20\x20\x20\x20\x20\x20'origin':\x20'https://www.google.com',\n\x20\x20\x20\x20\x20\x20'Sec-Fetch-Site':\x20'same-origin',\n\x20\x20\x20\x20\x20\x20'Sec-Fetch-Mode':\x20'navigate',\n\x20\x20\x20\x20\x20\x20'Sec-Fetch-User':\x20'?1',\n\x20\x20\x20\x20\x20\x20'Sec-Fetch-Dest':\x20'document'\n\x20\x20\x20\x20}\n\x20\x20\x20\x20#\x20代理\n\x20\x20\x20\x20proxies\x20=\x20{'http':\x20'http://127.0.0.1:7890',\x20'https':\x20'http://127.0.0.1:7890'}\n\x20\x20\x20\x20response\x20=\x20requests.get(url=url,\x20headers=headers,\x20proxies=proxies,\x20verify=False,\x20timeout=5)\n\x20\x20\x20\x20soup\x20=\x20BeautifulSoup(response.content,\x20'html.parser')\n\x20\x20\x20\x20#\x20查找全部div标签\n\x20\x20\x20\x20find_div\x20=\x20soup.find_all('div',\x20{'class':\x20'yuRUbf'})\n\x20\x20\x20\x20#\x20开启线程锁\n\x20\x20\x20\x20locks.acquire()\n\x20\x20\x20\x20#\x20加入列表\n\x20\x20\x20\x20get_url\x20=\x20[url.findNext('a')['href']\x20+\x20'\n'\x20for\x20url\x20in\x20find_div\x20if\x20'google.com'\x20not\x20in\x20url.findNext('a')['href']]\n\x20\x20\x20\x20global\x20url_num,\x20dork_finish_num\n\x20\x20\x20\x20url_num\x20+=\x20len(get_url)\n\x20\x20\x20\x20dork_finish_num\x20+=\x201\n\x20\x20\x20\x20print(Fore.GREEN\x20+\x20f'\r{now_time}[INFO]{\x20'-'\x20*\x2010}>get\x20Urlnumber：{url_num}\x20\x20Dorsk\x20number：{dork_finish_num}\x20/\x20{dork_total_num}',\x20end=''\x20+\x20Fore.RESET)\n\x20\x20\x20\x20#\x20写入文件\n\x20\x20\x20\x20write_info(filename,\x20get_url)\n\x20\x20\x20\x20#\x20释放线程锁\n\x20\x20\x20\x20locks.release()\n\n\x20\x20except\x20TimeoutError:\n\x20\x20\x20\x20pass\n\n\n#\x20写入文件函数\ndef\x20write_info(filename,\x20get_url):\n\x20\x20with\x20open(filename,\x20mode='a+',\x20encoding='utf-8')\x20as\x20file:\n\x20\x20\x20\x20file.writelines(get_url)\n\n\nif\x20__name__\x20==\x20'main':\n\x20\x20while\x20True:\n\x20\x20\x20\x20try:\n\x20\x20\x20\x20\x20\x20init()\x20#\x20初始化颜色模块\n\x20\x20\x20\x20\x20\x20dorks_file\x20=\x20input(Fore.YELLOW\x20+\x20f'\n{now_time}[INFO]{\x20'-'\x20*\x2010}>input\x20file:'\x20+\x20Fore.RESET)\n\x20\x20\x20\x20\x20\x20print('')\n\x20\x20\x20\x20\x20\x20filename\x20=\x20input(Fore.YELLOW\x20+\x20f'\n{now_time}[INFO]{\x20'-'\x20*\x2010}>output\x20file:'\x20+\x20Fore.RESET)\n\x20\x20\x20\x20\x20\x20#\x20接受work函数返回的元组\n\x20\x20\x20\x20\x20\x20dork_total_num,\x20query_list\x20=\x20work(dorks_file)\n\x20\x20\x20\x20\x20\x20#\x20定义全局变量完成数量/URL数量\n\x20\x20\x20\x20\x20\x20dork_finish_num\x20=\x20url_num\x20=\x200\n\n\x20\x20\x20\x20\x20\x20#\x20定义线程池数量\n\x20\x20\x20\x20\x20\x20threads\x20=\x20ThreadPoolExecutor(max_workers=20)\n\n\x20\x20\x20\x20\x20\x20#\x20定义全局锁\n\x20\x20\x20\x20\x20\x20threads_lock\x20=\x20Lock()\n\n\x20\x20\x20\x20\x20\x20#\x20分配进程池任务\n\x20\x20\x20\x20\x20\x20for\x20dokr_list\x20in\x20query_list:\n\x20\x20\x20\x20\x20\x20\x20\x20threads.submit(google_serach,\x20dokr_list,\x20threads_lock,\x20filename)\n\n\x20\x20\x20\x20\x20\x20threads.shutdown()\n\x20\x20\x20\x20\x20\x20if\x20len(sys.argv)\x20==\x201:\n\x20\x20\x20\x20\x20\x20\x20\x20pass\n\x20\x20\x20\x20\x20\x20input(Fore.YELLOW\x20+\x20f'\n\n{now_time}[INFO]{'-'\x20*\x2010}>final\x20huiche'\x20+\x20Fore.RESET)\n\x20\x20\x20\x20\x20\x20break\n\x20\x20\x20\x20#\x20文件为空\n\x20\x20\x20\x20except\x20FileNotFoundError:\n\x20\x20\x20\x20\x20\x20print(Fore.RED\x20+\x20f'{now_time}[Error]{'-'\x20*\x2010}>fiel\x20not\x20find'\x20+\x20Fore.RESET)\n\x20\x20\x20\x20#\x20中断异常\n\x20\x20\x20\x20except\x20KeyboardInterrupt:\n\x20\x20\x20\x20\x20\x20sys.exit(1)