From 24b4f9f5e25a04b6b7e27f5e5f9ab8b03ab0bae5 Mon Sep 17 00:00:00 2001 From: lededev Date: Mon, 18 Oct 2021 10:51:32 +0800 Subject: [PATCH] =?UTF-8?q?=E5=B0=86=E5=85=83=E6=95=B0=E6=8D=AE=E7=9A=84?= =?UTF-8?q?=E6=9D=A5=E6=BA=90=E7=BD=91=E7=AB=99=E8=AE=B0=E5=85=A5=E6=97=A5?= =?UTF-8?q?=E5=BF=97=E4=BB=A5=E4=BE=BF=E8=BF=9B=E8=A1=8C=E8=AF=84=E4=BC=B0?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- ADC_function.py | 4 ++-- WebCrawler/__init__.py | 2 ++ WebCrawler/storyline.py | 2 +- config.py | 4 ++-- 4 files changed, 7 insertions(+), 5 deletions(-) diff --git a/ADC_function.py b/ADC_function.py index e43fe5f..5b1d507 100755 --- a/ADC_function.py +++ b/ADC_function.py @@ -1,6 +1,6 @@ from os import replace import requests -import hashlib +#import hashlib from pathlib import Path import secrets import os.path @@ -20,7 +20,7 @@ def getXpathSingle(htmlcode, xpath): return result1 -G_USER_AGENT = r'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/94.0.4606.71 Safari/537.36' +G_USER_AGENT = r'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/94.0.4606.81 Safari/537.36' # 网页请求核心 def get_html(url, cookies: dict = None, ua: str = None, return_type: str = None): diff --git a/WebCrawler/__init__.py b/WebCrawler/__init__.py index c5d02b5..b6e7b2f 100644 --- a/WebCrawler/__init__.py +++ b/WebCrawler/__init__.py @@ -115,6 +115,7 @@ def get_data_from_json(file_number): # 从JSON返回元数据 json_data = json.loads(pool.apply_async(func_mapping[source], (file_number,)).get()) # if any service return a valid return, break if get_data_state(json_data): + print(f"[+]Find movie [{file_number}] metadata on website '{source}'") break pool.close() pool.terminate() @@ -126,6 +127,7 @@ def get_data_from_json(file_number): # 从JSON返回元数据 json_data = json.loads(func_mapping[source](file_number)) # if any service return a valid return, break if get_data_state(json_data): + print(f"[+]Find movie [{file_number}] metadata on website '{source}'") break except: break diff --git a/WebCrawler/storyline.py b/WebCrawler/storyline.py index b32778d..5ad4fd7 100644 --- a/WebCrawler/storyline.py +++ b/WebCrawler/storyline.py @@ -34,7 +34,7 @@ def getStoryline(number, title): for i in range(cnt): sl = len(result[i])if isinstance(result[i], str) else 0 if sl and first: - s += f'，[选中结果{apply_sites[i]}字数:{sl}]' + s += f'，[选中{apply_sites[i]}字数:{sl}]' first = False sel = result[i] elif sl: diff --git a/config.py b/config.py index 5624e85..abe030e 100644 --- a/config.py +++ b/config.py @@ -394,10 +394,10 @@ if __name__ == "__main__": code = compile(evstr, "", "eval") print('{}: "{}"'.format(evstr, eval(code))) config = Config() - mfilter = ('conf', 'proxy', '_exit', '_default_config', 'getboolean_override', 'getint_override', 'get_override', 'ini_path') + mfilter = {'conf', 'proxy', '_exit', '_default_config', 'getboolean_override', 'getint_override', 'get_override', 'ini_path'} for _m in [m for m in dir(config) if not m.startswith('__') and m not in mfilter]: evprint(f'config.{_m}()') - pfilter = ('proxies', 'SUPPORT_PROXY_TYPE') + pfilter = {'proxies', 'SUPPORT_PROXY_TYPE'} # test getInstance() assert(getInstance() == config) for _p in [p for p in dir(getInstance().proxy()) if not p.startswith('__') and p not in pfilter]: