From 5d4bc3454a282d16fc1ff03d216baf2fbc17a111 Mon Sep 17 00:00:00 2001 From: yobailover <38131170+yobailover@users.noreply.github.com> Date: Tue, 15 Sep 2020 00:05:17 +0800 Subject: [PATCH 1/2] =?UTF-8?q?=E6=94=B9=E7=88=B2=E4=BD=BF=E7=94=A8?= =?UTF-8?q?=F0=9F=97=BE=E6=97=A5=E6=9C=AC=E8=AA=9EMetadata=EF=BC=8C?= =?UTF-8?q?=E5=90=8C=E6=99=82=E5=B0=8D=E7=9B=A3=E4=BF=AE=E3=80=81Studio?= =?UTF-8?q?=E5=88=AE=E5=89=8A=E9=80=B2=E8=A1=8C=E7=B4=B0=E5=BE=AE=E8=AA=BF?= =?UTF-8?q?=E6=95=B4?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit 改爲使用🗾日本語Metadata，同時對監修、Studio刮削進行細微調整 --- WebCrawler/javbus.py | 55 ++++++++++++++++++++++++++++---------------- 1 file changed, 35 insertions(+), 20 deletions(-) diff --git a/WebCrawler/javbus.py b/WebCrawler/javbus.py index 42446e2..fe955b6 100644 --- a/WebCrawler/javbus.py +++ b/WebCrawler/javbus.py @@ -29,9 +29,16 @@ def getTitle(htmlcode): #获取标题 return title2 except: return title -def getStudio(htmlcode): #获取厂商 +def getStudio(htmlcode): #获取厂商 已修改 html = etree.fromstring(htmlcode,etree.HTMLParser()) - result = str(html.xpath('/html/body/div[5]/div[1]/div[2]/p[5]/a/text()')).strip(" ['']") + # 如果记录中冇导演，厂商排在第4位 + if 'メーカー:' == str(html.xpath('/html/body/div[5]/div[1]/div[2]/p[4]/span/text()')).strip(" ['']"): + result = str(html.xpath('/html/body/div[5]/div[1]/div[2]/p[4]/a/text()')).strip(" ['']") + # 如果记录中有导演，厂商排在第5位 + elif 'メーカー:' == str(html.xpath('/html/body/div[5]/div[1]/div[2]/p[5]/span/text()')).strip(" ['']"): + result = str(html.xpath('/html/body/div[5]/div[1]/div[2]/p[5]/a/text()')).strip(" ['']") + else: + result = '' return result def getYear(htmlcode): #获取年份 html = etree.fromstring(htmlcode,etree.HTMLParser()) @@ -45,10 +52,10 @@ def getRelease(htmlcode): #获取出版日期 html = etree.fromstring(htmlcode, etree.HTMLParser()) result = str(html.xpath('/html/body/div[5]/div[1]/div[2]/p[2]/text()')).strip(" ['']") return result -def getRuntime(htmlcode): #获取分钟 - soup = BeautifulSoup(htmlcode, 'lxml') - a = soup.find(text=re.compile('分鐘')) - return a +def getRuntime(htmlcode): #获取分钟 已修改 + html = etree.fromstring(htmlcode, etree.HTMLParser()) + result = str(html.xpath('/html/body/div[5]/div[1]/div[2]/p[3]/text()')).strip(" ['']分鐘") + return result def getActor(htmlcode): #获取女优 b=[] soup=BeautifulSoup(htmlcode,'lxml') @@ -60,9 +67,12 @@ def getNum(htmlcode): #获取番号 html = etree.fromstring(htmlcode, etree.HTMLParser()) result = str(html.xpath('/html/body/div[5]/div[1]/div[2]/p[1]/span[2]/text()')).strip(" ['']") return result -def getDirector(htmlcode): #获取导演 +def getDirector(htmlcode): #获取导演 已修改 html = etree.fromstring(htmlcode, etree.HTMLParser()) - result = str(html.xpath('/html/body/div[5]/div[1]/div[2]/p[4]/a/text()')).strip(" ['']") + if '監督:' == str(html.xpath('/html/body/div[5]/div[1]/div[2]/p[4]/span/text()')).strip(" ['']"): + result = str(html.xpath('/html/body/div[5]/div[1]/div[2]/p[4]/a/text()')).strip(" ['']") + else: + result = '' # 记录中有可能没有导演数据 return result def getCID(htmlcode): html = etree.fromstring(htmlcode, etree.HTMLParser()) @@ -77,14 +87,18 @@ def getOutline(htmlcode): #获取演员 return result except: return '' -def getSerise(htmlcode): - try: - html = etree.fromstring(htmlcode, etree.HTMLParser()) +def getSerise(htmlcode): #获取系列 已修改 + html = etree.fromstring(htmlcode, etree.HTMLParser()) + # 如果记录中冇导演，系列排在第6位 + if 'シリーズ:' == str(html.xpath('/html/body/div[5]/div[1]/div[2]/p[6]/span/text()')).strip(" ['']"): + result = str(html.xpath('/html/body/div[5]/div[1]/div[2]/p[6]/a/text()')).strip(" ['']") + # 如果记录中有导演，系列排在第7位 + elif 'シリーズ:' == str(html.xpath('/html/body/div[5]/div[1]/div[2]/p[7]/span/text()')).strip(" ['']"): result = str(html.xpath('/html/body/div[5]/div[1]/div[2]/p[7]/a/text()')).strip(" ['']") - return result - except: - return '' -def getTag(htmlcode): # 获取演员 + else: + result = '' + return result +def getTag(htmlcode): # 获取标签 tag = [] soup = BeautifulSoup(htmlcode, 'lxml') a = soup.find_all(attrs={'class': 'genre'}) @@ -94,10 +108,11 @@ def getTag(htmlcode): # 获取演员 tag.append(i.get_text()) return tag + def main_uncensored(number): - htmlcode = get_html('https://www.javbus.com/' + number) + htmlcode = get_html('https://www.javbus.com/ja/' + number) if getTitle(htmlcode) == '': - htmlcode = get_html('https://www.javbus.com/' + number.replace('-','_')) + htmlcode = get_html('https://www.javbus.com/ja/' + number.replace('-','_')) try: dww_htmlcode = fanza.main_htmlcode(getCID(htmlcode)) except: @@ -117,7 +132,7 @@ def main_uncensored(number): 'label': getSerise(htmlcode), 'imagecut': 0, 'actor_photo': '', - 'website': 'https://www.javbus.com/' + number, + 'website': 'https://www.javbus.com/ja/' + number, 'source': 'javbus.py', 'series': getSerise(htmlcode), } @@ -128,7 +143,7 @@ def main_uncensored(number): def main(number): try: try: - htmlcode = get_html('https://www.javbus.com/' + number) + htmlcode = get_html('https://www.javbus.com/ja/' + number) try: dww_htmlcode = fanza.main_htmlcode(getCID(htmlcode)) except: @@ -148,7 +163,7 @@ def main(number): 'tag': getTag(htmlcode), 'label': getSerise(htmlcode), 'actor_photo': getActorPhoto(htmlcode), - 'website': 'https://www.javbus.com/' + number, + 'website': 'https://www.javbus.com/ja/' + number, 'source': 'javbus.py', 'series': getSerise(htmlcode), } From 3421d1ecbf2c4fe017469e4a4c34eca9bdd1503c Mon Sep 17 00:00:00 2001 From: yobailover <38131170+yobailover@users.noreply.github.com> Date: Tue, 15 Sep 2020 00:21:03 +0800 Subject: [PATCH 2/2] =?UTF-8?q?=E7=82=BAStudio=E6=96=B0=E5=A2=9E=E6=98=A0?= =?UTF-8?q?=E5=B0=84=E8=A1=A8=EF=BC=8C=E5=B0=87=E5=B8=B8=E8=A6=8B=E7=89=87?= =?UTF-8?q?=E5=81=87=E5=90=8D=E6=98=A0=E5=B0=84=E7=82=BA=E8=8B=B1=E6=96=87?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit * 為Studio新增映射表，將常見片假名映射為英文 * 修改封面图裁剪算式 --- core.py | 39 ++++++++++++++++++++++++++++++++++++++- 1 file changed, 38 insertions(+), 1 deletion(-) diff --git a/core.py b/core.py index 49bf590..9ccdbd9 100755 --- a/core.py +++ b/core.py @@ -152,6 +152,43 @@ def get_data_from_json(file_number, filepath, conf: config.Config): # 从JSON cover_small = tmpArr[0].strip('\"').strip('\'') # ====================处理异常字符 END================== #\/:*?"<>| + # === 替换Studio片假名 + studio = studio.replace('アイエナジー','Energy') + studio = studio.replace('アイデアポケット','Idea Pocket') + studio = studio.replace('アキノリ','AKNR') + studio = studio.replace('アタッカーズ','Attackers') + studio = re.sub('アパッチ.*','Apache',studio) + studio = studio.replace('アマチュアインディーズ','SOD') + studio = studio.replace('アリスJAPAN','Alice Japan') + studio = studio.replace('オーロラプロジェクト・アネックス','Aurora Project Annex') + studio = studio.replace('クリスタル映像','Crystal 映像') + studio = studio.replace('グローリークエスト','Glory Quest') + studio = studio.replace('ダスッ！','DAS！') + studio = studio.replace('ディープス','DEEP’s') + studio = studio.replace('ドグマ','Dogma') + studio = studio.replace('プレステージ','PRESTIGE') + studio = studio.replace('ムーディーズ','MOODYZ') + studio = studio.replace('メディアステーション','宇宙企画') + studio = studio.replace('ワンズファクトリー','WANZ FACTORY') + studio = studio.replace('エスワン ナンバーワンスタイル','S1') + studio = studio.replace('エスワンナンバーワンスタイル','S1') + studio = studio.replace('SODクリエイト','SOD') + studio = studio.replace('サディスティックヴィレッジ','SOD') + studio = studio.replace('V＆Rプロダクツ','V＆R PRODUCE') + studio = studio.replace('V＆RPRODUCE','V＆R PRODUCE') + studio = studio.replace('レアルワークス','Real Works') + studio = studio.replace('マックスエー','MAX-A') + studio = studio.replace('ピーターズMAX','PETERS MAX') + studio = studio.replace('プレミアム','PREMIUM') + studio = studio.replace('ナチュラルハイ','NATURAL HIGH') + studio = studio.replace('マキシング','MAXING') + studio = studio.replace('エムズビデオグループ','M’s Video Group') + studio = studio.replace('ミニマム','Minimum') + studio = studio.replace('ワープエンタテインメント','WAAP Entertainment') + studio = re.sub('.*/妄想族','妄想族',studio) + studio = studio.replace('/',' ') + # === 替换Studio片假名 END + location_rule = eval(conf.location_rule()) # Process only Windows. @@ -357,7 +394,7 @@ def cutImage(imagecut, path, number, c_word): imgSize = img.size w = img.width h = img.height - img2 = img.crop((w / 1.9, 0, w, h)) + img2 = img.crop((w - h / 1.5, 0, w, h)) img2.save(path + '/' + number + c_word + '-poster.jpg') print('[+]Image Cutted! ' + path + '/' + number + c_word + '-poster.jpg') except: