diff --git a/.vscode/launch.json b/.vscode/launch.json index 978418a..16ae9dd 100644 --- a/.vscode/launch.json +++ b/.vscode/launch.json @@ -5,8 +5,26 @@ "name": "AV_Data_Capture", "type": "python", "request": "launch", + "console": "integratedTerminal", + "env": { + "PYTHONIOENCODING": "utf-8" + }, "program": "${workspaceFolder}/AV_Data_capture.py", - "console": "integratedTerminal" + "program1": "${workspaceFolder}/WebCrawler/javbus.py", + "program2": "${workspaceFolder}/WebCrawler/javdb.py", + "program3": "${workspaceFolder}/WebCrawler/xcity.py", + "program4": "${workspaceFolder}/number_parser.py", + "program5": "${workspaceFolder}/config.py", + "cwd0": "${fileDirname}", + "cwd1": "${workspaceFolder}/dist", + "cwd2": "${env:HOME}${env:USERPROFILE}/.avdc", + "args0": ["-a","-p","J:/Downloads","-o","J:/log"], + "args1": ["-g","-m","3","-c","1","-d","0"], + "args2": ["-igd0", "-m3", "-p", "J:/JAV_output", "-q", "121220_001"], + "args3": ["-agd0","-m3", "-q", ".*","-p","J:/#JAV_output3"], + "args4": ["-gic1", "-d0", "-m3", "-o", "avlog", "-p", "I:/JAV_output"], + "args5": ["-gic1", "-d0", "-m1", "-o", "avlog", "-p", "J:/Downloads"], + "args6": ["-z", "-o", "J:/log"] } ] -} \ No newline at end of file +} diff --git a/AV_Data_Capture.py b/AV_Data_Capture.py index 7483a8d..778cd03 100755 --- a/AV_Data_Capture.py +++ b/AV_Data_Capture.py @@ -35,7 +35,7 @@ def check_update(local_version): print("[*]======================================================") -def argparse_function(ver: str) -> typing.Tuple[str, str, bool]: +def argparse_function(ver: str) -> typing.Tuple[str, str, str, str, bool]: conf = config.getInstance() parser = argparse.ArgumentParser(epilog=f"Load Config file '{conf.ini_path}'.") parser.add_argument("file", default='', nargs='?', help="Single Movie file path.") @@ -61,7 +61,7 @@ def argparse_function(ver: str) -> typing.Tuple[str, str, bool]: help="""Only show job list of files and numbers, and **NO** actual operation is performed. It may help you correct wrong numbers before real job.""") parser.add_argument("-v", "--version", action="version", version=ver) - #ini_path + args = parser.parse_args() def get_natural_number_or_none(value): return int(value) if isinstance(value, str) and value.isnumeric() and int(value)>=0 else None diff --git a/WebCrawler/avsox.py b/WebCrawler/avsox.py index e38a452..c6e6f00 100644 --- a/WebCrawler/avsox.py +++ b/WebCrawler/avsox.py @@ -58,7 +58,7 @@ def getCover_small(html): return result def getTag(html): x = html.xpath('/html/head/meta[@name="keywords"]/@content')[0].split(',') - return x[2:] if len(x) > 2 else [] + return [translateTag_to_sc(i.strip()) for i in x[2:]] if len(x) > 2 else [] def getSeries(html): try: result1 = str(html.xpath('//span[contains(text(),"系列:")]/../span[2]/text()')).strip(" ['']") diff --git a/WebCrawler/storyline.py b/WebCrawler/storyline.py index 9b0a44c..4fcca0e 100644 --- a/WebCrawler/storyline.py +++ b/WebCrawler/storyline.py @@ -16,7 +16,7 @@ G_mode_txt = ('顺序执行','线程池','进程池') class noThread(object): def map(self, fn, param): - return builtins.map(fn, param) + return list(builtins.map(fn, param)) def __enter__(self): return self def __exit__(self, exc_type, exc_val, exc_tb): @@ -34,11 +34,14 @@ def getStoryline(number, title, sites: list=None): else: storyine_sites += conf.storyline_censored_site().split(',') r_dup = set() - apply_sites = [] + sort_sites = [] for s in storyine_sites: - if s in G_registered_storyline_site and s not in r_dup: - apply_sites.append(s) - r_dup.add(s) + ns = re.sub(r'.*?:', '', s, re.A) + if ns in G_registered_storyline_site and ns not in r_dup: + sort_sites.append(s) + r_dup.add(ns) + sort_sites.sort() + apply_sites = [re.sub(r'.*?:', '', s, re.A) for s in sort_sites] mp_args = ((site, number, title, debug) for site in apply_sites) cores = min(len(apply_sites), os.cpu_count()) if cores == 0: @@ -47,7 +50,6 @@ def getStoryline(number, title, sites: list=None): assert run_mode in (0,1,2) with ThreadPool(cores) if run_mode == 1 else Pool(cores) if run_mode == 2 else noThread() as pool: result = pool.map(getStoryline_mp, mp_args) - result = list(result) if run_mode == 0 else result if not debug and conf.storyline_show() == 0: for value in result: if isinstance(value, str) and len(value): @@ -126,7 +128,7 @@ def getStoryline_airav(number, debug): return desc except Exception as e: if debug: - print(f"[-]MP getOutline_amazon Error: {e},number [{number}].") + print(f"[-]MP getStoryline_airav Error: {e},number [{number}].") pass return None diff --git a/config.ini b/config.ini index 9ff2bb8..24404c6 100755 --- a/config.ini +++ b/config.ini @@ -89,15 +89,15 @@ extrafanart_folder=extrafanart ; 剧情简介 [storyline] ; website为javbus javdb avsox xcity carib时,site censored_site uncensored_site 为获取剧情简介信息的 -; 可选数据源站点列表。列表内站点同时并发查询,取值优先级从左到右,靠左站点没数据才会采用后面站点获得的。 +; 可选数据源站点列表。列表内站点同时并发查询,取值优先级由冒号前的序号决定,从小到大,数字小的站点没数据才会采用后面站点获得的。 ; 其中airav avno1 58avgo是中文剧情简介,区别是airav只能查有码,avno1有码无码都能查,58avgo只能查无码或者 ; 流出破解马赛克的影片(此功能没使用)。 ; xcity和amazon是日语的,由于amazon商城没有番号信息,选中对应DVD的准确率仅99.6%。如果三个列表全部为空则不查询, ; 设置成不查询可大幅提高刮削速度。 ; site= -site=avno1 -censored_site=airav,xcity,amazon -uncensored_site=58avgo +site=3:avno1 +censored_site=1:airav,4:xcity,5:amazon +uncensored_site=2:58avgo ; 运行模式:0:顺序执行(最慢) 1:线程池(默认值) 2:进程池(启动开销比线程池大,并发站点越多越快) run_mode=1 ; show_result剧情简介调试信息 0关闭 1简略 2详细(详细部分不记入日志),剧情简介失效时可打开2查看原因