From 4e96d572047d30eb85dc1abd6f75961f4b41c629 Mon Sep 17 00:00:00 2001 From: lededev Date: Sat, 23 Oct 2021 18:14:25 +0800 Subject: [PATCH 1/8] extrafanart download speed 6x up by thread pool --- config.ini | 1 + config.py | 4 ++++ core.py | 46 +++++++++++++++++++++++++++++++++++++++++++--- 3 files changed, 48 insertions(+), 3 deletions(-) diff --git a/config.ini b/config.ini index 9ff2bb8..ccf4dc1 100755 --- a/config.ini +++ b/config.ini @@ -84,6 +84,7 @@ water=2 ; 剧照 [extrafanart] switch=0 +parallel_download=1 extrafanart_folder=extrafanart ; 剧情简介 diff --git a/config.py b/config.py index f6d6488..142a0f7 100644 --- a/config.py +++ b/config.py @@ -153,6 +153,9 @@ class Config: def is_extrafanart(self) -> bool: return self.conf.getboolean("extrafanart", "switch") + def extrafanart_thread_pool_download(self) -> bool: + return self.conf.getboolean("extrafanart", "parallel_download") + def watermark_type(self) -> int: return int(self.conf.get("watermark", "water")) @@ -363,6 +366,7 @@ class Config: conf.add_section(sec13) conf.set(sec13, "switch", 1) conf.set(sec13, "extrafanart_folder", "extrafanart") + conf.set(sec13, "parallel_download", 1) sec14 = "storyline" conf.add_section(sec14) diff --git a/core.py b/core.py index 24c1ce5..73f2e5a 100755 --- a/core.py +++ b/core.py @@ -9,6 +9,7 @@ from PIL import Image from io import BytesIO from pathlib import Path from datetime import datetime +from concurrent.futures import ThreadPoolExecutor from ADC_function import * from WebCrawler import get_data_from_json @@ -181,7 +182,13 @@ def trailer_download(trailer, leak_word, c_word, number, path, filepath): print('[+]Video Downloaded!', path + '/' + number + leak_word + c_word + '-trailer.mp4') # 剧照下载成功，否则移动到failed -def extrafanart_download(data, path, filepath): +def extrafanart_download(data, path, number, filepath): + if config.getInstance().extrafanart_thread_pool_download(): + return extrafanart_download_threadpool(data, path, number, filepath) + extrafanart_download_one_by_one(data, path, filepath) + +def extrafanart_download_one_by_one(data, path, filepath): + tm_start = time.perf_counter() j = 1 conf = config.getInstance() path = os.path.join(path, conf.get_extrafanart()) @@ -206,8 +213,41 @@ def extrafanart_download(data, path, filepath): return print('[+]Image Downloaded!', jpg_fullpath) j += 1 + if conf.debug(): + print(f'[!]Extrafanart download one by one mode runtime {time.perf_counter() - tm_start:.3f}s') +def download_one_file(args): + return _download_one_file(*args) +def _download_one_file(url: str, save_path: Path): + filebytes = get_html(url, return_type='content') + if isinstance(filebytes, bytes) and len(filebytes): + if len(filebytes) == save_path.open('wb').write(filebytes): + return f'[+]Image Downloaded! {save_path}' + return None + +def extrafanart_download_threadpool(url_list, save_dir, number, source_movie): + tm_start = time.perf_counter() + conf = config.getInstance() + extrafanart_dir = Path(save_dir) / conf.get_extrafanart() + download_only_missing_images = conf.download_only_missing_images() + mp_args = [] + for i in range(len(url_list)): + jpg_filename = f'extrafanart-{i+1}.jpg' + jpg_fullpath = extrafanart_dir / jpg_filename + if download_only_missing_images and not file_not_exist_or_empty(jpg_fullpath): + continue + mp_args.append((url_list[i], jpg_fullpath)) + if not len(mp_args): + return + with ThreadPoolExecutor(os.cpu_count()) as pool: + result = pool.map(download_one_file, mp_args) + for s in sorted(result, key=lambda p: 0 if not p else int(re.findall('(\d+)\.jpg$', p, re.A)[0])): + print(s) if s else None + if not all(result): # 非致命错误，电影不移入失败文件夹，将来可以用模式3补齐 + print('[-]Failed download some extrafanart images for number [{number}], you may retry run mode 3 later.') + if conf.debug(): + print(f'[!]Extrafanart download ThreadPool mode runtime {time.perf_counter() - tm_start:.3f}s') # 封面是否下载成功，否则移动到failed def image_download(cover, number, leak_word, c_word, path, filepath): @@ -610,7 +650,7 @@ def core_main(file_path, number_th): try: # 下载剧照 data, path, filepath if conf.is_extrafanart() and json_data.get('extrafanart'): - extrafanart_download(json_data.get('extrafanart'), path, filepath) + extrafanart_download(json_data.get('extrafanart'), path, number, filepath) except: pass @@ -658,7 +698,7 @@ def core_main(file_path, number_th): # 下载剧照 data, path, filepath if conf.is_extrafanart() and json_data.get('extrafanart'): - extrafanart_download(json_data.get('extrafanart'), path, filepath) + extrafanart_download(json_data.get('extrafanart'), path, number, filepath) # 裁剪图 cutImage(imagecut, path, number, leak_word, c_word) From 03954c3a3564771d08184b8d4debbbd43d0c04bd Mon Sep 17 00:00:00 2001 From: lededev Date: Sat, 23 Oct 2021 19:19:38 +0800 Subject: [PATCH 2/8] map return result keeping the same order with args, no need sorted() --- core.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/core.py b/core.py index 73f2e5a..cf9d040 100755 --- a/core.py +++ b/core.py @@ -242,7 +242,7 @@ def extrafanart_download_threadpool(url_list, save_dir, number, source_movie): return with ThreadPoolExecutor(os.cpu_count()) as pool: result = pool.map(download_one_file, mp_args) - for s in sorted(result, key=lambda p: 0 if not p else int(re.findall('(\d+)\.jpg$', p, re.A)[0])): + for s in result: print(s) if s else None if not all(result): # 非致命错误，电影不移入失败文件夹，将来可以用模式3补齐 print('[-]Failed download some extrafanart images for number [{number}], you may retry run mode 3 later.') From db1ad1d582a78e0b40eef4669ecae3bc0298af50 Mon Sep 17 00:00:00 2001 From: lededev Date: Sat, 23 Oct 2021 19:28:20 +0800 Subject: [PATCH 3/8] make download_one_file() more general --- core.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/core.py b/core.py index cf9d040..fdf4898 100755 --- a/core.py +++ b/core.py @@ -223,7 +223,7 @@ def _download_one_file(url: str, save_path: Path): filebytes = get_html(url, return_type='content') if isinstance(filebytes, bytes) and len(filebytes): if len(filebytes) == save_path.open('wb').write(filebytes): - return f'[+]Image Downloaded! {save_path}' + return str(save_path) return None def extrafanart_download_threadpool(url_list, save_dir, number, source_movie): @@ -243,7 +243,7 @@ def extrafanart_download_threadpool(url_list, save_dir, number, source_movie): with ThreadPoolExecutor(os.cpu_count()) as pool: result = pool.map(download_one_file, mp_args) for s in result: - print(s) if s else None + print('[+]Image Downloaded!', s) if s else None if not all(result): # 非致命错误，电影不移入失败文件夹，将来可以用模式3补齐 print('[-]Failed download some extrafanart images for number [{number}], you may retry run mode 3 later.') if conf.debug(): From 0758104e7a63dcd490c9b963135c11732e4599eb Mon Sep 17 00:00:00 2001 From: lededev Date: Sat, 23 Oct 2021 19:42:36 +0800 Subject: [PATCH 4/8] output error message also --- core.py | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/core.py b/core.py index fdf4898..6803c25 100755 --- a/core.py +++ b/core.py @@ -241,11 +241,11 @@ def extrafanart_download_threadpool(url_list, save_dir, number, source_movie): if not len(mp_args): return with ThreadPoolExecutor(os.cpu_count()) as pool: - result = pool.map(download_one_file, mp_args) - for s in result: - print('[+]Image Downloaded!', s) if s else None + result = list(pool.map(download_one_file, mp_args)) + for i in range(len(result)): + print('[+]Extrafanart Downloaded!', result[i]) if result[i] else print(f'[-]Extrafanart {i+1} for [{number}] download failed!') if not all(result): # 非致命错误，电影不移入失败文件夹，将来可以用模式3补齐 - print('[-]Failed download some extrafanart images for number [{number}], you may retry run mode 3 later.') + print('[-]Failed download some extrafanart images for [{number}], you may retry run mode 3 later.') if conf.debug(): print(f'[!]Extrafanart download ThreadPool mode runtime {time.perf_counter() - tm_start:.3f}s') From 9c5db258ecd38b0386f4710e41b145e083ab9db6 Mon Sep 17 00:00:00 2001 From: lededev Date: Sat, 23 Oct 2021 20:42:27 +0800 Subject: [PATCH 5/8] clean up unused param --- core.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/core.py b/core.py index 6803c25..3e1cd91 100755 --- a/core.py +++ b/core.py @@ -184,7 +184,7 @@ def trailer_download(trailer, leak_word, c_word, number, path, filepath): # 剧照下载成功，否则移动到failed def extrafanart_download(data, path, number, filepath): if config.getInstance().extrafanart_thread_pool_download(): - return extrafanart_download_threadpool(data, path, number, filepath) + return extrafanart_download_threadpool(data, path, number) extrafanart_download_one_by_one(data, path, filepath) def extrafanart_download_one_by_one(data, path, filepath): @@ -226,7 +226,7 @@ def _download_one_file(url: str, save_path: Path): return str(save_path) return None -def extrafanart_download_threadpool(url_list, save_dir, number, source_movie): +def extrafanart_download_threadpool(url_list, save_dir, number): tm_start = time.perf_counter() conf = config.getInstance() extrafanart_dir = Path(save_dir) / conf.get_extrafanart() From e8eb3ff1921726ac7be778ce422d0d29490acaed Mon Sep 17 00:00:00 2001 From: lededev Date: Sat, 23 Oct 2021 21:05:49 +0800 Subject: [PATCH 6/8] change download threads number configable default parallel_download=5 --- config.ini | 2 +- config.py | 8 ++++++-- core.py | 5 ++++- 3 files changed, 11 insertions(+), 4 deletions(-) diff --git a/config.ini b/config.ini index ccf4dc1..4c9aced 100755 --- a/config.ini +++ b/config.ini @@ -84,7 +84,7 @@ water=2 ; 剧照 [extrafanart] switch=0 -parallel_download=1 +parallel_download=5 extrafanart_folder=extrafanart ; 剧情简介 diff --git a/config.py b/config.py index 142a0f7..138c9e1 100644 --- a/config.py +++ b/config.py @@ -153,8 +153,12 @@ class Config: def is_extrafanart(self) -> bool: return self.conf.getboolean("extrafanart", "switch") - def extrafanart_thread_pool_download(self) -> bool: - return self.conf.getboolean("extrafanart", "parallel_download") + def extrafanart_thread_pool_download(self) -> int: + try: + v = self.conf.getint("extrafanart", "parallel_download") + return v if v >= 0 else 5 + except: + return 5 def watermark_type(self) -> int: return int(self.conf.get("watermark", "water")) diff --git a/core.py b/core.py index 3e1cd91..0bf306b 100755 --- a/core.py +++ b/core.py @@ -240,7 +240,10 @@ def extrafanart_download_threadpool(url_list, save_dir, number): mp_args.append((url_list[i], jpg_fullpath)) if not len(mp_args): return - with ThreadPoolExecutor(os.cpu_count()) as pool: + parallel = min(len(mp_args), conf.extrafanart_thread_pool_download()) + if parallel > 100: + print('[!]Warrning: Parallel download thread too large may cause website ban IP!') + with ThreadPoolExecutor(parallel) as pool: result = list(pool.map(download_one_file, mp_args)) for i in range(len(result)): print('[+]Extrafanart Downloaded!', result[i]) if result[i] else print(f'[-]Extrafanart {i+1} for [{number}] download failed!') From 1cf1782a22b8dd79484c60beec18c29260a3ec8f Mon Sep 17 00:00:00 2001 From: lededev Date: Sun, 24 Oct 2021 01:04:23 +0800 Subject: [PATCH 7/8] bug fix using pathlib mkdir -p --- core.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/core.py b/core.py index 0bf306b..c3a488e 100755 --- a/core.py +++ b/core.py @@ -233,13 +233,13 @@ def extrafanart_download_threadpool(url_list, save_dir, number): download_only_missing_images = conf.download_only_missing_images() mp_args = [] for i in range(len(url_list)): - jpg_filename = f'extrafanart-{i+1}.jpg' - jpg_fullpath = extrafanart_dir / jpg_filename + jpg_fullpath = extrafanart_dir / f'extrafanart-{i+1}.jpg' if download_only_missing_images and not file_not_exist_or_empty(jpg_fullpath): continue mp_args.append((url_list[i], jpg_fullpath)) if not len(mp_args): return + extrafanart_dir.mkdir(parents=True, exist_ok=True) parallel = min(len(mp_args), conf.extrafanart_thread_pool_download()) if parallel > 100: print('[!]Warrning: Parallel download thread too large may cause website ban IP!') From 2f523ea540f4b7dbdb1f3b0483b219a6e1e7e465 Mon Sep 17 00:00:00 2001 From: lededev Date: Sun, 24 Oct 2021 01:31:24 +0800 Subject: [PATCH 8/8] =?UTF-8?q?=E5=89=A7=E7=85=A7=E4=B8=8B=E8=BD=BD:?= =?UTF-8?q?=E5=8F=AA=E6=98=BE=E7=A4=BA=E6=AF=8F=E4=B8=AA=E9=94=99=E8=AF=AF?= =?UTF-8?q?,=E5=A6=82=E6=9E=9C=E5=85=A8=E9=83=A8=E6=88=90=E5=8A=9F?= =?UTF-8?q?=E5=8F=AA=E6=98=BE=E7=A4=BA=E4=B8=80=E8=A1=8C=E6=B1=87=E6=80=BB?= =?UTF-8?q?,=E5=A4=A7=E5=B9=85=E7=B2=BE=E7=AE=80=E6=97=A5=E5=BF=97?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- core.py | 9 +++++++-- 1 file changed, 7 insertions(+), 2 deletions(-) diff --git a/core.py b/core.py index c3a488e..0e662d1 100755 --- a/core.py +++ b/core.py @@ -245,10 +245,15 @@ def extrafanart_download_threadpool(url_list, save_dir, number): print('[!]Warrning: Parallel download thread too large may cause website ban IP!') with ThreadPoolExecutor(parallel) as pool: result = list(pool.map(download_one_file, mp_args)) + failed = 0 for i in range(len(result)): - print('[+]Extrafanart Downloaded!', result[i]) if result[i] else print(f'[-]Extrafanart {i+1} for [{number}] download failed!') + if not result[i]: + print(f'[-]Extrafanart {i+1} for [{number}] download failed!') + failed += 1 if not all(result): # 非致命错误，电影不移入失败文件夹，将来可以用模式3补齐 - print('[-]Failed download some extrafanart images for [{number}], you may retry run mode 3 later.') + print(f"[-]Failed downloaded {failed}/{len(result)} extrafanart images for [{number}] to '{extrafanart_dir}', you may retry run mode 3 later.") + else: + print(f"[+]Successfully downloaded {len(result)} extrafanart to '{extrafanart_dir}'") if conf.debug(): print(f'[!]Extrafanart download ThreadPool mode runtime {time.perf_counter() - tm_start:.3f}s')