class SeleniumUtil def __init__self driver selfdriver = driver def quitself 页面瑞出 selfdriverquit def refreshself 页面刷新 timesleep5 selfdriverrefresh
根据代码逻辑分析,问题出在EmailIngestionProcessor类中的process方法中,因为在每次执行完任务后会调用self.spider.quit()关闭driver,导致后续执行任务时无法正常使用driver。
解决方法是将self.spider.quit()移动到start方法的finally块中,在所有任务执行完后再关闭driver。具体修改如下:
class EmailIngestionProcessor(DFMProcessor):
def __init__(self):
DFMProcessor.__init__(self)
self.lock = threading.Lock()
self.name = "EmailIngestionProcessor"
self.spider = EmailSpider(self.driver)
self.execute_sql = ExecuteSql()
def process(self):
self.spider.get_advanced_and_ingestion()
self.spider.quit() # 将关闭driver的操作移动到process方法中
def run():
processor = EmailIngestionProcessor()
processor.start()
processor.spider.quit() # 将关闭driver的操作移动到run方法中
然后在DFMProcessor类中的start方法中,也要将self.spider.quit()移动到finally块中,确保所有任务执行完后再关闭driver。具体修改如下:
class DFMProcessor(BaseProcessor):
def __init__(self):
BaseProcessor.__init__(self)
self.name = 'DFMProcessor'
self.lock = ''
self.display = settings['CHROME_DISPLAY']
self.driver = get_chrome_driver(self.get_download_path(), display=self.display)
self.spider = BaseSpider(self.driver)
def process(self):
pass
def get_download_path(self):
return ''
def start(self):
try:
self.lock.acquire()
logger.info(r" >>> 开始执行 {}".format(self.name))
self.spider.check_login()
self.process()
except Exception as e:
log_error(e)
logger.error(r" >>> 执行 {} 异常: {}".format(self.name, e))
finally:
logger.info(r" >>> 完成执行 {}".format(self.name))
self.spider.quit() # 将关闭driver的操作移动到finally块中
requests.session().close() # 关闭连接池
time.sleep(10)
self.lock.release()
这样修改后,每次执行完任务后会关闭driver,确保后续任务能够正常执行
原文地址: https://www.cveoy.top/t/topic/iEQL 著作权归作者所有。请勿转载和采集!