From 54d8f3af87bea96e1aa260c1af507f54217fe44e Mon Sep 17 00:00:00 2001 From: lededev Date: Sun, 6 Mar 2022 01:31:43 +0800 Subject: [PATCH 1/9] madou priority against javdb --- WebCrawler/__init__.py | 4 ++++ WebCrawler/madou.py | 5 +++-- 2 files changed, 7 insertions(+), 2 deletions(-) diff --git a/WebCrawler/__init__.py b/WebCrawler/__init__.py index d467eab..f5ce63c 100644 --- a/WebCrawler/__init__.py +++ b/WebCrawler/__init__.py @@ -95,6 +95,10 @@ def get_data_from_json(file_number, oCC): "rj" in lo_file_number or "vj" in lo_file_number ): sources.insert(0, sources.pop(sources.index("dlsite"))) + elif "madou" in sources and ( + re.match(r"^md[0-9]{4}$", lo_file_number) + ): + sources.insert(0, sources.pop(sources.index("madou"))) elif re.match(r"^[a-z0-9]{3,}$", lo_file_number): if "javdb" in sources: sources.insert(0, sources.pop(sources.index("javdb"))) diff --git a/WebCrawler/madou.py b/WebCrawler/madou.py index 01fc19c..9dc7773 100644 --- a/WebCrawler/madou.py +++ b/WebCrawler/madou.py @@ -1,3 +1,5 @@ +import sys +sys.path.append('../') from bs4 import BeautifulSoup # need install from lxml import etree # need install from pyquery import PyQuery as pq # need install @@ -5,10 +7,8 @@ from ADC_function import * import json import re from lib2to3.pgen2 import parse -import sys from urllib.parse import urlparse, unquote -sys.path.append('../') def getActorPhoto(html): @@ -162,3 +162,4 @@ def main(number): if __name__ == '__main__': print(main('MD0094')) + print(main('MD0222')) From 48d14e19ae881be3afcdcb804587670e20854268 Mon Sep 17 00:00:00 2001 From: lededev Date: Sun, 6 Mar 2022 01:37:09 +0800 Subject: [PATCH 2/9] add madou and mv91 to website list --- config.ini | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/config.ini b/config.ini index e611762..740b050 100755 --- a/config.ini +++ b/config.ini @@ -41,7 +41,7 @@ max_title_len=50 update_check=1 [priority] -website=javbus,airav,fanza,xcity,javdb,mgstage,fc2,avsox,dlsite,carib,fc2club +website=javbus,airav,fanza,xcity,javdb,mgstage,fc2,avsox,dlsite,carib,fc2club,madou,mv91 [escape] literals=\()/ @@ -116,4 +116,4 @@ sites=33,34 ; 人脸识别 hog:方向梯度直方图(不太准确，速度快) cnn:深度学习模型(准确，需要GPU/CUDA,速度慢) [face] -locations_model=hog \ No newline at end of file +locations_model=hog From 1cecf66a843a2fbd24554aaa49b448bf6caf099f Mon Sep 17 00:00:00 2001 From: lededev Date: Sun, 6 Mar 2022 16:08:37 +0800 Subject: [PATCH 3/9] support more madou numbers --- WebCrawler/__init__.py | 10 ++++++---- 1 file changed, 6 insertions(+), 4 deletions(-) diff --git a/WebCrawler/__init__.py b/WebCrawler/__init__.py index f5ce63c..6e431b8 100644 --- a/WebCrawler/__init__.py +++ b/WebCrawler/__init__.py @@ -95,15 +95,17 @@ def get_data_from_json(file_number, oCC): "rj" in lo_file_number or "vj" in lo_file_number ): sources.insert(0, sources.pop(sources.index("dlsite"))) - elif "madou" in sources and ( - re.match(r"^md[0-9]{4}$", lo_file_number) - ): - sources.insert(0, sources.pop(sources.index("madou"))) elif re.match(r"^[a-z0-9]{3,}$", lo_file_number): if "javdb" in sources: sources.insert(0, sources.pop(sources.index("javdb"))) if "xcity" in sources: sources.insert(0, sources.pop(sources.index("xcity"))) + if "madou" in sources: + sources.insert(0, sources.pop(sources.index("madou"))) + elif "madou" in sources and ( + re.match(r"^[a-z0-9]{3,}-[0-9]{2,}$", lo_file_number) + ): + sources.insert(0, sources.pop(sources.index("madou"))) # check sources in func_mapping todel = [] From 788fc4a97c528f1815c04b73ce7d0a3fec15b68f Mon Sep 17 00:00:00 2001 From: lededev Date: Sun, 6 Mar 2022 16:22:11 +0800 Subject: [PATCH 4/9] madou numbers MD0140-2 MD0165-8 --- WebCrawler/__init__.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/WebCrawler/__init__.py b/WebCrawler/__init__.py index 6e431b8..7f9cf19 100644 --- a/WebCrawler/__init__.py +++ b/WebCrawler/__init__.py @@ -103,7 +103,7 @@ def get_data_from_json(file_number, oCC): if "madou" in sources: sources.insert(0, sources.pop(sources.index("madou"))) elif "madou" in sources and ( - re.match(r"^[a-z0-9]{3,}-[0-9]{2,}$", lo_file_number) + re.match(r"^[a-z0-9]{3,}-[0-9]{1,}$", lo_file_number) ): sources.insert(0, sources.pop(sources.index("madou"))) From 31da166931ff38cf94fd8c4867714f19803a8f9c Mon Sep 17 00:00:00 2001 From: lededev Date: Sun, 6 Mar 2022 17:29:15 +0800 Subject: [PATCH 5/9] madou.py:fix get title for MD0140-2 --- WebCrawler/madou.py | 12 +++++------- 1 file changed, 5 insertions(+), 7 deletions(-) diff --git a/WebCrawler/madou.py b/WebCrawler/madou.py index 9dc7773..be0f5b1 100644 --- a/WebCrawler/madou.py +++ b/WebCrawler/madou.py @@ -16,12 +16,10 @@ def getActorPhoto(html): def getTitle(html, number): # 获取标题 - title = str(html.xpath('//h1[@class="article-title"]/text()')[0]) - try: - result = str(re.split(r'[/|／|-]', title)[1]) - return result.strip() - except: - return title.replace(number.upper(), '').strip() + # MD0140-2 / 家有性事EP2 爱在身边-麻豆社 + browser_title = str(html.xpath("/html/head/title/text()")[0]) + browser_title = browser_title[browser_title.find('/ ') + 2:].strip() + return browser_title[:browser_title.find('-麻豆社')].strip() def getStudio(html): # 获取厂商 已修改 @@ -161,5 +159,5 @@ def main(number): if __name__ == '__main__': - print(main('MD0094')) print(main('MD0222')) + print(main('MD0140-2')) From 3117b3a18d09a706ac562c55bdf1f5dca77db767 Mon Sep 17 00:00:00 2001 From: lededev Date: Sun, 6 Mar 2022 17:36:22 +0800 Subject: [PATCH 6/9] madou.py:fix get title for MAD039 --- WebCrawler/madou.py | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/WebCrawler/madou.py b/WebCrawler/madou.py index be0f5b1..0c03cf7 100644 --- a/WebCrawler/madou.py +++ b/WebCrawler/madou.py @@ -17,8 +17,9 @@ def getActorPhoto(html): def getTitle(html, number): # 获取标题 # MD0140-2 / 家有性事EP2 爱在身边-麻豆社 + # MAD039 机灵可爱小叫花 强诱僧人迫犯色戒-麻豆社 browser_title = str(html.xpath("/html/head/title/text()")[0]) - browser_title = browser_title[browser_title.find('/ ') + 2:].strip() + browser_title = browser_title[browser_title.find(' '):].replace('/','').strip() return browser_title[:browser_title.find('-麻豆社')].strip() @@ -161,3 +162,4 @@ def main(number): if __name__ == '__main__': print(main('MD0222')) print(main('MD0140-2')) + print(main('MAD039')) From 8ad49973428021722a163038b4b7a206a39d8bd2 Mon Sep 17 00:00:00 2001 From: lededev Date: Sun, 6 Mar 2022 20:39:59 +0800 Subject: [PATCH 7/9] madou.py:simp by regex --- WebCrawler/madou.py | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/WebCrawler/madou.py b/WebCrawler/madou.py index 0c03cf7..1e10481 100644 --- a/WebCrawler/madou.py +++ b/WebCrawler/madou.py @@ -19,8 +19,7 @@ def getTitle(html, number): # 获取标题 # MD0140-2 / 家有性事EP2 爱在身边-麻豆社 # MAD039 机灵可爱小叫花 强诱僧人迫犯色戒-麻豆社 browser_title = str(html.xpath("/html/head/title/text()")[0]) - browser_title = browser_title[browser_title.find(' '):].replace('/','').strip() - return browser_title[:browser_title.find('-麻豆社')].strip() + return str(re.findall(r'^.*?( / | )(.*)-麻豆社$', browser_title)[0][1]).strip() def getStudio(html): # 获取厂商 已修改 From 6b7e518fbe1716f3f1ba1fd1ebe9d08a4af3d896 Mon Sep 17 00:00:00 2001 From: lededev Date: Sun, 6 Mar 2022 21:03:00 +0800 Subject: [PATCH 8/9] madou.py:fix get tags --- WebCrawler/madou.py | 16 +++++++++++----- 1 file changed, 11 insertions(+), 5 deletions(-) diff --git a/WebCrawler/madou.py b/WebCrawler/madou.py index 1e10481..fffd08b 100644 --- a/WebCrawler/madou.py +++ b/WebCrawler/madou.py @@ -81,13 +81,15 @@ def getSerise(html): # 获取系列 已修改 return '' -def getTag(html): # 获取标签 - return html.xpath('//div[@class="article-tags"]/a/text()') +def getTag(html, studio): # 获取标签 + x = html.xpath('/html/head/meta[@name="keywords"]/@content')[0].split(',') + return [i.strip() for i in x if len(i.strip()) and studio not in i and '麻豆' not in i] def getExtrafanart(html): # 获取剧照 return '' + def cutTags(tags): actors = [] tags = [] @@ -107,13 +109,15 @@ def main(number): html = etree.fromstring(htmlcode, etree.HTMLParser()) url = getUrl(html) - tags = getTag(html) - actor,tags = cutTags(tags); + studio = getStudio(html) + tags = getTag(html, studio) + #actor,tags = cutTags(tags) # 演员在tags中的位置不固定，放弃尝试获取 + actor = '' dic = { # 标题 'title': getTitle(html, number), # 制作商 - 'studio': getStudio(html), + 'studio': studio, # 年份 'year': getYear(html), # 简介 @@ -162,3 +166,5 @@ if __name__ == '__main__': print(main('MD0222')) print(main('MD0140-2')) print(main('MAD039')) + print(main('JDMY027')) + From b6786ef9d71df32ca245f08cf6e259096e58d5f8 Mon Sep 17 00:00:00 2001 From: lededev Date: Sun, 6 Mar 2022 21:20:20 +0800 Subject: [PATCH 9/9] fc2.py:fix some pages can not auto detect UTF-8 encoding --- WebCrawler/fc2.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/WebCrawler/fc2.py b/WebCrawler/fc2.py index c559c8d..6885ce5 100644 --- a/WebCrawler/fc2.py +++ b/WebCrawler/fc2.py @@ -84,7 +84,7 @@ def getTrailer(htmlcode, number): def main(number): try: number = number.replace('FC2-', '').replace('fc2-', '') - htmlcode2 = ADC_function.get_html('https://adult.contents.fc2.com/article/' + number + '/') + htmlcode2 = ADC_function.get_html('https://adult.contents.fc2.com/article/' + number + '/', encoding='utf-8') actor = getActor_fc2com(htmlcode2) if not actor: actor = '素人' @@ -123,4 +123,5 @@ def main(number): if __name__ == '__main__': print(main('FC2-1787685')) print(main('FC2-2086710')) + print(main('FC2-2182382'))