Feature/check duplicate

config.yml

@@ -21,6 +21,8 @@ scanner:
  minimum_size: 232MiB
  skip_nfo_dir: yes
  manual: yes
+ # 扫描影片文件时，使用索引文件跳过已整理的影片 （避免重复整理）
+ ignore_file_in_index: true
 ################################
 network:
  # 设置代理服务器地址，支持 http, socks5/socks5h 代理，比如'http://127.0.0.1:1080'
@@ -72,7 +74,9 @@ summarizer:
  move_files: true
 
  # 路径相关的选项
- path: 
+ path:
+ # 以整理文件的索引文件，存放在input_directory中
+ summarizer_index_file: 'summarizer_index.txt'
  # 存放影片、封面等文件的文件夹路径
  output_folder_pattern: '#整理完成/{actress}/[{num}] {title}'
  # 影片、封面、nfo信息文件等的文件名将基于下面的规则来创建

javsp/__main__.py

@@ -502,10 +502,12 @@ def check_step(result, msg='步骤错误'):
  elapsed = time.strftime("%M:%S", time.gmtime(info['elapsed']))
  speed = get_fmt_size(info['rate']) + '/s'
  logger.info(f"已下载剧照{pic_url} {id}.png: {width}x{height}, {filesize} [{elapsed}, {speed}]")
- else:
+ else: 
  check_step(False, f"下载剧照{id}: {pic_url}失败")
  except:
- check_step(False, f"下载剧照{id}: {pic_url}失败")
+ # 下载失败，不要在rasise exception，跳过，从而不打断整理
+ logger.error(f"下载剧照{id}: {pic_url}失败")
+ # check_step(False, f"下载剧照{id}: {pic_url}失败")
  time.sleep(scrape_interval)
  check_step(True)
 
@@ -519,6 +521,10 @@ def check_step(result, msg='步骤错误'):
  logger.info(f'整理完成，相关文件已保存到: {movie.save_dir}\n')
  else:
  logger.info(f'刮削完成，相关文件已保存到: {movie.nfo_file}\n')
+
+ # 将整理好的影片写入到索引文件中
+ if Cfg().summarizer.path.summarizer_index_file: 
+ write_summarizer_index_file(os.getcwd(), [movie])
 
  if movie != all_movies[-1] and Cfg().crawler.sleep_after_scraping > Duration(0):
  time.sleep(Cfg().crawler.sleep_after_scraping.total_seconds())

javsp/config.py

@@ -17,7 +17,7 @@ class Scanner(BaseConfig):
  minimum_size: ByteSize
  skip_nfo_dir: bool
  manual: bool
-
+ ignore_file_in_index: bool 
 class CrawlerID(str, Enum):
  airav = 'airav'
  avsox = 'avsox'
@@ -125,6 +125,7 @@ class MovieDefault(BaseConfig):
  publisher: str
 
 class PathSummarize(BaseConfig):
+ summarizer_index_file: str
  output_folder_pattern: str
  basename_pattern: str
  length_maximum: PositiveInt

javsp/file.py

@@ -10,7 +10,7 @@
 from typing import List
 
 
-__all__ = ['scan_movies', 'get_fmt_size', 'get_remaining_path_len', 'replace_illegal_chars', 'get_failed_when_scan', 'find_subtitle_in_dir']
+__all__ = ['scan_movies', 'get_fmt_size', 'get_remaining_path_len', 'replace_illegal_chars', 'get_failed_when_scan', 'find_subtitle_in_dir', 'write_summarizer_index_file']
 
 
 from javsp.avid import *
@@ -22,6 +22,22 @@
 failed_items = []
 
 
+def load_summarizer_index_file(root: str):
+ """加载索引文件"""
+ index_file = os.path.join(root, Cfg().summarizer.path.summarizer_index_file)
+ try:
+ with open(index_file, 'r', encoding='utf-8') as f:
+ return [line.rstrip('\n') for line in f.readlines()]
+ except FileNotFoundError:
+ return []
+
+def write_summarizer_index_file(root: str, movies: List[Movie]):
+ """写入索引文件"""
+ index_file = os.path.join(root, Cfg().summarizer.path.summarizer_index_file)
+ with open(index_file, 'a+', encoding='utf-8') as f:
+ for movie in movies:
+ f.write(movie.files[0] + '\n')
+
 def scan_movies(root: str) -> List[Movie]:
  """获取文件夹内的所有影片的列表（自动探测同一文件夹内的分片）"""
  # 由于实现的限制: 
@@ -31,6 +47,7 @@ def scan_movies(root: str) -> List[Movie]:
  # 扫描所有影片文件并获取它们的番号
  dic = {} # avid: [abspath1, abspath2...]
  small_videos = {}
+ files_in_index = [] if not Cfg().scanner.ignore_file_in_index else load_summarizer_index_file(root)
  ignore_folder_name_pattern = re.compile('|'.join(Cfg().scanner.ignored_folder_name_pattern))
  for dirpath, dirnames, filenames in os.walk(root):
  for name in dirnames.copy():
@@ -51,6 +68,10 @@ def scan_movies(root: str) -> List[Movie]:
  if filesize < Cfg().scanner.minimum_size:
  small_videos.setdefault(file, []).append(fullpath)
  continue
+ # 如果文件名在索引文件中，则跳过
+ if Cfg().scanner.ignore_file_in_index:
+ if fullpath in files_in_index:
+ continue
  dvdid = get_id(fullpath)
  cid = get_cid(fullpath)
  # 如果文件名能匹配到cid，那么将cid视为有效id，因为此时dvdid多半是错的