Merge branch 'testing' with new id_fix and db_export

This commit is contained in:
simon 2021-05-22 13:21:51 +07:00
commit 118ba8f664
3 changed files with 375 additions and 337 deletions

View File

@ -15,13 +15,13 @@ import src.trailers as trailers
import src.id_fix as id_fix
def get_pending_all(config):
def get_pending_all():
""" figure out what needs to be done """
# call subfunction to collect pending
pending_movie = moviesort.MovieHandler().pending
pending_tv = tvsort.TvHandler().pending
pending_trailer = len(trailers.TrailerHandler().pending)
pending_movie_fix = len(id_fix.get_pending(config))
pending_movie_fix = len(id_fix.MovieNameFix().pending)
pending_total = pending_movie + pending_tv + pending_trailer + pending_movie_fix
# build dict
pending = {}
@ -69,32 +69,32 @@ def print_menu(stdscr, current_row_idx, menu, pending):
stdscr.refresh()
def sel_handler(menu_item, config):
def sel_handler(menu_item):
""" lunch scripts from here based on selection """
if menu_item == 'All':
moviesort.main()
tvsort.main()
db_export.main(config)
db_export.main()
trailers.main()
id_fix.main(config)
id_fix.main()
elif menu_item == 'Movies':
moviesort.main()
elif menu_item == 'TV shows':
tvsort.main()
elif menu_item == 'DB export':
db_export.main(config)
db_export.main()
elif menu_item == 'Trailer download':
trailers.main()
elif menu_item == 'Fix Movie Names':
id_fix.main(config)
id_fix.main()
def curses_main(stdscr, menu, config):
def curses_main(stdscr, menu):
""" curses main to desplay and restart the menu """
curses.curs_set(0)
curses.init_pair(1, curses.COLOR_BLUE, curses.COLOR_WHITE)
current_row_idx = 0
pending = get_pending_all(config)
pending = get_pending_all()
print_menu(stdscr, current_row_idx, menu, pending)
# endless loop
while True:
@ -132,14 +132,14 @@ def main():
logging.basicConfig(filename=log_file,level=logging.INFO,format='%(asctime)s:%(message)s')
# endless loop
while True:
pending = get_pending_all(config)
pending = get_pending_all()
if not pending:
return
menu_item = curses.wrapper(curses_main, menu, config)
menu_item = curses.wrapper(curses_main, menu)
if menu_item == 'Exit':
return
else:
sel_handler(menu_item, config)
sel_handler(menu_item)
sleep(3)

View File

@ -2,238 +2,237 @@
import csv
from os import path
import requests
def get_items(config):
""" get json from emby """
emby_url = config['emby']['emby_url']
emby_user_id = config['emby']['emby_user_id']
emby_api_key = config['emby']['emby_api_key']
# movies
url = (f'{emby_url}/Users/{emby_user_id}/Items?api_key={emby_api_key}' +
'&Recursive=true&IncludeItemTypes=Movie' +
'&fields=Genres,MediaStreams,Overview,ProviderIds' +
'&SortBy=DateCreated&SortOrder=Descending')
r = requests.get(url)
all_movies = r.json()['Items']
# episodes
url = (f'{emby_url}/Users/{emby_user_id}/Items?api_key={emby_api_key}' +
'&IncludeItemTypes=Episode&Recursive=true&StartIndex=0' +
'&Fields=DateCreated,Genres,MediaStreams,MediaSources,Overview,ProviderIds'
'&SortBy=DateCreated&SortOrder=Descending&IsMissing=false')
r = requests.get(url)
all_episodes = r.json()['Items']
return all_movies, all_episodes
from src.config import get_config
def parse_movies(all_movies):
""" loop through the movies """
movie_info_csv = []
movie_tech_csv = []
movie_seen = []
for movie in all_movies:
# general
movie_name = movie['Name']
overview = movie['Overview']
imdb = movie['ProviderIds']['Imdb']
played = movie['UserData']['Played']
genres = ', '.join(movie['Genres'])
# media
for i in movie['MediaSources']:
if i['Protocol'] == 'File':
file_name = path.basename(i['Path'])
year = path.splitext(file_name)[0][-5:-1]
duration_min = round(i['RunTimeTicks'] / 600000000)
filesize_MB = round(i['Size'] / 1024 / 1024)
for j in i['MediaStreams']:
if j['Type'] == 'Video':
image_width = j['Width']
image_height = j['Height']
avg_bitrate_MB = round(j['BitRate'] / 1024 / 1024, 2)
codec = j['Codec']
# found it
break
# found it
break
# info csv
info_dict = {}
info_dict['movie_name'] = movie_name
info_dict['year'] = year
info_dict['imdb'] = imdb
info_dict['genres'] = genres
info_dict['overview'] = overview
info_dict['duration_min'] = duration_min
movie_info_csv.append(info_dict)
# technical csv
tech_dict = {}
tech_dict['file_name'] = file_name
tech_dict['duration_min'] = duration_min
tech_dict['filesize_MB'] = filesize_MB
tech_dict['image_width'] = image_width
tech_dict['image_height'] = image_height
tech_dict['avg_bitrate_MB'] = avg_bitrate_MB
tech_dict['codec'] = codec
movie_tech_csv.append(tech_dict)
# seen or unseen
if played == True:
icon = '[X]'
elif played == False:
icon = '[ ]'
seen_line = f'{icon} {movie_name} ({year})'
movie_seen.append(seen_line)
return movie_info_csv, movie_tech_csv, movie_seen
class DatabaseExport():
""" saves database to CSV """
CONFIG = get_config()
def __init__(self):
self.all_movies, self.all_episodes = self.get_items()
def get_items(self):
""" get json from emby """
emby_url = self.CONFIG['emby']['emby_url']
emby_user_id = self.CONFIG['emby']['emby_user_id']
emby_api_key = self.CONFIG['emby']['emby_api_key']
# movies
url = (f'{emby_url}/Users/{emby_user_id}/Items?api_key={emby_api_key}'
'&Recursive=true&IncludeItemTypes=Movie'
'&fields=Genres,MediaStreams,Overview,'
'ProviderIds,Path,RunTimeTicks'
'&SortBy=DateCreated&SortOrder=Descending')
response = requests.get(url)
all_movies = response.json()['Items']
# episodes
url = (f'{emby_url}/Users/{emby_user_id}/Items?api_key={emby_api_key}'
'&IncludeItemTypes=Episode&Recursive=true&StartIndex=0'
'&Fields=DateCreated,Genres,MediaStreams,'
'MediaSources,Overview,ProviderIds,Path,RunTimeTicks'
'&SortBy=DateCreated&SortOrder=Descending&IsMissing=false')
response = requests.get(url)
all_episodes = response.json()['Items']
return all_movies, all_episodes
def parse_movies(self):
""" handle the movies """
all_movies = self.all_movies
# seen
movie_seen = ListParser.build_seen(all_movies)
self.write_seen(movie_seen, 'movienew')
# tech
movie_tech = ListParser.build_tech(all_movies)
self.write_csv(movie_tech, 'movie-tech.csv')
# info
movie_info = ListParser.build_movie_info(all_movies)
self.write_csv(movie_info, 'movie-info.csv')
def parse_episodes(self):
""" handle the episodes """
all_episodes = self.all_episodes
# seen
episode_seen = ListParser.build_seen(all_episodes)
self.write_seen(episode_seen, 'episodenew')
# tech
episode_tech = ListParser.build_tech(all_episodes)
self.write_csv(episode_tech, 'episode-tech.csv')
# info
episode_info = ListParser.build_episode_info(all_episodes)
self.write_csv(episode_info, 'episode-info.csv')
def write_csv(self, to_write, filename):
""" write list of dicts to CSV """
log_folder = self.CONFIG['media']['log_folder']
file_path = path.join(log_folder, filename)
# open and write
with open(file_path, 'w') as f:
# take fieldnames from first line
fieldnames = to_write[0].keys()
csv_writer = csv.DictWriter(f, fieldnames)
csv_writer.writeheader()
csv_writer.writerows(to_write)
def write_seen(self, to_write, filename):
""" write list of seen """
log_folder = self.CONFIG['media']['log_folder']
file_path = path.join(log_folder, filename)
# movie by new
with open(file_path, 'w') as f:
for line in to_write:
f.write(line + '\n')
def write_movie_files(movie_info_csv, movie_tech_csv, movie_seen, config):
""" writes the csv files to disk """
log_folder = config['media']['log_folder']
class ListParser():
""" static parse the lists from DatabaseExport """
# movie info
movie_info_sorted = sorted(movie_info_csv, key=lambda k: k['movie_name'])
file_path = path.join(log_folder, 'movie-info.csv')
# open and write
with open(file_path, 'w') as f:
# take fieldnames from first line
fieldnames = movie_info_sorted[0].keys()
csv_writer = csv.DictWriter(f, fieldnames)
csv_writer.writeheader()
csv_writer.writerows(movie_info_sorted)
@staticmethod
def build_seen(filelist):
""" build the seen list """
# movie tech
movie_tech_csv_sorted = sorted(movie_tech_csv, key=lambda k: k['file_name'])
file_path = path.join(log_folder, 'movie-tech.csv')
# open and write
with open(file_path, 'w') as f:
# take fieldnames from first line
fieldnames = movie_tech_csv_sorted[0].keys()
csv_writer = csv.DictWriter(f, fieldnames)
csv_writer.writeheader()
csv_writer.writerows(movie_tech_csv_sorted)
# movie by new
file_path = path.join(log_folder, 'movienew')
with open(file_path, 'w') as f:
for line in movie_seen:
f.write(line + '\n')
file_item_seen = []
for file_item in filelist:
played = file_item['UserData']['Played']
file_name = path.basename(file_item['Path'])
file_item_name = path.splitext(file_name)[0]
# seen or unseen
if played:
icon = '[X]'
else:
icon = '[ ]'
seen_line = f'{icon} {file_item_name}'
file_item_seen.append(seen_line)
return file_item_seen
@staticmethod
def build_tech(filelist):
""" build tech csv """
file_item_tech = []
for file_item in filelist:
file_name = path.basename(file_item['Path'])
duration_min = round(file_item['RunTimeTicks'] / 600000000)
# loop through media sources
for i in file_item['MediaSources']:
if i['Protocol'] == 'File':
filesize = round(i['Size'] / 1024 / 1024)
for j in i['MediaStreams']:
if j['Type'] == 'Video':
image_width = j['Width']
image_height = j['Height']
avg_bitrate = round(j['BitRate'] / 1024 / 1024, 2)
codec = j['Codec']
# found it
break
# found it
break
# technical csv
tech_dict = {}
tech_dict['file_name'] = file_name
tech_dict['duration_min'] = duration_min
tech_dict['filesize_MB'] = filesize
tech_dict['image_width'] = image_width
tech_dict['image_height'] = image_height
tech_dict['avg_bitrate_MB'] = avg_bitrate
tech_dict['codec'] = codec
file_item_tech.append(tech_dict)
# sort and return
file_item_tech_sorted = sorted(
file_item_tech, key=lambda k: k['file_name']
)
return file_item_tech_sorted
@staticmethod
def build_movie_info(all_movies):
""" build movie info csv """
movie_info = []
for movie in all_movies:
movie_name = movie['Name']
year = movie['Path'].split('/')[3]
overview = movie['Overview']
imdb = movie['ProviderIds']['Imdb']
genres = ', '.join(movie['Genres'])
duration_min = round(movie['RunTimeTicks'] / 600000000)
info_dict = {}
info_dict['movie_name'] = movie_name
info_dict['year'] = year
info_dict['imdb'] = imdb
info_dict['genres'] = genres
info_dict['overview'] = overview
info_dict['duration_min'] = duration_min
movie_info.append(info_dict)
# sort and return
movie_info_sorted = sorted(movie_info, key=lambda k: k['movie_name'])
return movie_info_sorted
@staticmethod
def build_episode_info(all_episodes):
""" build episode info csv """
episode_info = []
for episode in all_episodes:
episode_name = episode['Name']
file_name = path.basename(episode['Path'])
try:
episode_id = episode['IndexNumber']
except KeyError:
# not a real episode
continue
try:
overview = episode['Overview'].replace('\n\n', ' ')
overview = overview.replace('\n', ' ')
except KeyError:
overview = 'NA'
try:
imdb = episode['ProviderIds']['Imdb']
except KeyError:
imdb = 'NA'
genres = ', '.join(episode['Genres'])
season_name = episode['SeasonName']
series_name = episode['SeriesName']
duration_min = round(episode['RunTimeTicks'] / 600000000)
# info csv
info_dict = {}
info_dict['series_name'] = series_name
info_dict['season_name'] = season_name
info_dict['episode_id'] = episode_id
info_dict['episode_name'] = episode_name
info_dict['file_name'] = file_name
info_dict['imdb'] = imdb
info_dict['genres'] = genres
info_dict['overview'] = overview
info_dict['duration_min'] = duration_min
episode_info.append(info_dict)
# sort and return
episode_info_sorted = sorted(
episode_info, key=lambda k: k['file_name']
)
return episode_info_sorted
def parse_episodes(all_episodes):
""" loop through all episodes """
episode_info_csv = []
episode_tech_csv = []
episode_seen = []
for episode in all_episodes:
if episode['ParentIndexNumber'] == 0:
# not a real episode
continue
# general
episode_name = episode['Name']
episode_id = episode['IndexNumber']
try:
overview = episode['Overview'].replace('\n\n', ' ').replace('\n', ' ')
except KeyError:
overview = 'NA'
try:
imdb = episode['ProviderIds']['Imdb']
except KeyError:
imdb = 'NA'
played = episode['UserData']['Played']
genres = ', '.join(episode['Genres'])
season_name = episode['SeasonName']
series_name = episode['SeriesName']
# media
for i in episode['MediaSources']:
if i['Protocol'] == 'File':
file_name = path.basename(i['Path'])
file_id = i['Name']
duration_min = round(i['RunTimeTicks'] / 600000000)
filesize_MB = round(i['Size'] / 1024 / 1024)
for j in i['MediaStreams']:
if j['Type'] == 'Video':
image_width = j['Width']
image_height = j['Height']
avg_bitrate_MB = round(j['BitRate'] / 1024 / 1024, 2)
codec = j['Codec']
# found it
break
# found it
break
# info csv
info_dict = {}
info_dict['series_name'] = series_name
info_dict['file_id'] = file_id
info_dict['season_name'] = season_name
info_dict['episode_id'] = episode_id
info_dict['episode_name'] = episode_name
info_dict['imdb'] = imdb
info_dict['genres'] = genres
info_dict['overview'] = overview
info_dict['duration_min'] = duration_min
episode_info_csv.append(info_dict)
# technical csv
tech_dict = {}
tech_dict['file_name'] = file_name
tech_dict['duration_min'] = duration_min
tech_dict['filesize_MB'] = filesize_MB
tech_dict['image_width'] = image_width
tech_dict['image_height'] = image_height
tech_dict['avg_bitrate_MB'] = avg_bitrate_MB
tech_dict['codec'] = codec
episode_tech_csv.append(tech_dict)
# seen or unseen
if played == True:
icon = '[X]'
elif played == False:
icon = '[ ]'
seen_line = f'{icon} {file_id}'
episode_seen.append(seen_line)
return episode_info_csv, episode_tech_csv, episode_seen
def write_episode_files(episode_info_csv, episode_tech_csv, episode_seen, config):
""" writes the csv files to disk """
log_folder = config['media']['log_folder']
# episode info
episode_info_sorted = sorted(episode_info_csv, key=lambda k: k['file_id'])
for i in episode_info_sorted:
i.pop('file_id', None)
file_path = path.join(log_folder, 'episode-info.csv')
# open and write
with open(file_path, 'w') as f:
# take fieldnames from first line
fieldnames = episode_info_sorted[0].keys()
csv_writer = csv.DictWriter(f, fieldnames)
csv_writer.writeheader()
csv_writer.writerows(episode_info_sorted)
# episode tech
episode_tech_csv_sorted = sorted(episode_tech_csv, key=lambda k: k['file_name'])
file_path = path.join(log_folder, 'episode-tech.csv')
# open and write
with open(file_path, 'w') as f:
# take fieldnames from first line
fieldnames = episode_tech_csv_sorted[0].keys()
csv_writer = csv.DictWriter(f, fieldnames)
csv_writer.writeheader()
csv_writer.writerows(episode_tech_csv_sorted)
# episode by new
file_path = path.join(log_folder, 'episodenew')
with open(file_path, 'w') as f:
for line in episode_seen:
f.write(line + '\n')
def main(config):
""" write collection to csv """
def main():
""" main to regenerate csv files """
print('recreating db files')
# get data
all_movies, all_episodes = get_items(config)
# write movies
movie_info_csv, movie_tech_csv, movie_seen = parse_movies(all_movies)
write_movie_files(movie_info_csv, movie_tech_csv, movie_seen, config)
# write episodes
episode_info_csv, episode_tech_csv, episode_seen = parse_episodes(all_episodes)
write_episode_files(episode_info_csv, episode_tech_csv, episode_seen, config)
export = DatabaseExport()
export.parse_movies()
export.parse_episodes()

View File

@ -2,119 +2,158 @@
import os
import re
import requests
import subprocess
from time import sleep
import requests
def get_emby_list(config):
""" get current emby movie list """
emby_url = config['emby']['emby_url']
emby_user_id = config['emby']['emby_user_id']
emby_api_key = config['emby']['emby_api_key']
url = (emby_url + '/Users/' + emby_user_id + '/Items?api_key=' + emby_api_key +
'&Recursive=True&IncludeItemTypes=Movie&Fields=Path,PremiereDate')
r_emby = requests.get(url).json()
movie_list = r_emby['Items']
return movie_list
from src.config import get_config
def compare_list(movie_list):
""" compare the movie_list and look for wong ids """
errors_list = []
for movie in movie_list:
# from file name
file_name = os.path.basename(os.path.splitext(movie['Path'])[0])
year_id_pattern = re.compile(r'\(\d{4}\)')
year_str = year_id_pattern.findall(file_name)[-1]
file_year = year_str.replace('(', '').replace(')', '')
file_base_name = file_name.replace(year_str, '').strip()
# dedected in emby
movie_name = movie['Name']
try:
premier_year = movie['PremiereDate'].split('-')[0]
except KeyError:
premier_year = file_year
# check for error
error = False
if file_base_name != movie_name:
for i, j in enumerate(file_base_name):
if j != movie_name[i] and j != '-':
class MovieNameFix():
""" check movie names in library and
rename if premiere date doesn't match with filename """
CONFIG = get_config()
def __init__(self):
self.movie_list = self.get_emby_list()
self.pending = self.find_errors()
def get_emby_list(self):
""" get current emby movie list """
emby_url = self.CONFIG['emby']['emby_url']
emby_user_id = self.CONFIG['emby']['emby_user_id']
emby_api_key = self.CONFIG['emby']['emby_api_key']
url = (f'{emby_url}/Users/{emby_user_id}/Items?api_key={emby_api_key}'
'&Recursive=True&IncludeItemTypes=Movie'
'&Fields=Path,PremiereDate')
request = requests.get(url).json()
movie_list = request['Items']
return movie_list
def find_errors(self):
""" find missmatch in movie_list """
errors = []
for movie in self.movie_list:
# parse filename
file_name = os.path.basename(movie['Path'])
ext = os.path.splitext(file_name)[1]
movie_name = os.path.splitext(file_name)[0]
year_id_pattern = re.compile(r'\((\d{4})\)$')
file_year = year_id_pattern.findall(movie_name)[-1]
movie_name_file = movie_name.split(f'({file_year})')[0].strip()
# premier date
try:
premier_year = movie['PremiereDate'].split('-')[0]
except KeyError:
premier_year = file_year
# emby
emby_name = movie['Name']
error = False
if emby_name != movie_name_file:
diff = self.str_diff(emby_name, movie_name_file)
if diff:
error = True
break
if file_year != premier_year:
error = True
# add to list on error
if error:
new_name = f'{movie_name} ({premier_year})'.replace('/', '-')
old = {'filename': file_name, 'year': file_year}
new = {'filename': new_name, 'year': premier_year}
errors_list.append([old, new])
return errors_list
if premier_year != file_year:
error = True
if error:
error_dict = {}
error_dict['old_year'] = file_year
error_dict['old_name'] = file_name
error_dict['new_year'] = premier_year
error_dict['new_name'] = f'{emby_name} ({premier_year}){ext}'
errors.append(error_dict)
return errors
@staticmethod
def str_diff(str1, str2):
""" simple diff calculator between two strings
ignoreing - and / """
diff = []
for num, value in enumerate(str1):
try:
if value not in (str2[num], '/'):
diff.append(value)
except IndexError:
diff.append(value)
for num, value in enumerate(str2):
try:
if value not in (str1[num], '-'):
diff.append(value)
except IndexError:
diff.append(value)
return list(diff)
def rename(config, errors_list):
""" rename files with correct names """
print(f'renaming {len(errors_list)} movies.')
moviepath = config['media']['moviepath']
skipped = []
for movie in errors_list:
old_year = movie[0]['year']
old_name = movie[0]['filename']
old_folder = os.path.join(moviepath, old_year, old_name)
def fix_errors(self):
""" select what to do """
skipped = []
fixed = []
print(f'found {len(self.pending)} problems')
for error in self.pending:
old_name = error['old_name']
new_name = error['new_name']
# prompt
print(f'\nrenaming from-to:\n{old_name}\n{new_name}')
print('[0]: skip')
print('[1]: rename')
print('[c]: cancel')
select = input()
rename_files = os.listdir(old_folder)
new_year = movie[1]['year']
new_name = movie[1]['filename']
# prompt
print(f'\nrenaming from-to:\n{old_name}\n{new_name}')
print('[0]: skip')
print('[1]: rename')
print('[c]: cancel')
select = input()
if select == 0:
skipped.append(old_name)
break
elif select == 'c':
return
# continue
for item in rename_files:
old_file_name = os.path.join(old_folder, item)
new_file_name = os.path.join(old_folder, item.replace(old_name, new_name))
os.rename(old_file_name, new_file_name)
# movie folder
os.rename(old_folder, old_folder.replace(old_name, new_name))
# year folder
if select == '1':
self.rename_files(error)
fixed.append(new_name)
elif select == '0':
skipped.append(old_name)
continue
elif select == 'c':
print('cancel')
return
else:
print(f'{select} is invalid input')
return
# pritty output
if skipped:
print('skipped files:')
for i in skipped:
print(i)
if fixed:
print(f'fixed {len(fixed)} movies')
def rename_files(self, error):
""" actually rename the files """
moviepath = self.CONFIG['media']['moviepath']
old_year = error['old_year']
new_year = error['new_year']
old_movie = os.path.splitext(error['old_name'])[0]
old_folder = os.path.join(moviepath, old_year, old_movie)
new_movie = os.path.splitext(error['new_name'])[0]
# handle folder
if old_year != new_year:
old_folder_name = old_folder.replace(f'({old_year})', f'({new_year})')
new_folder_name = old_folder_name.replace(old_year, new_year)
os.rename(old_folder_name, new_folder_name)
return skipped
old_year_folder = os.path.split(old_folder)[0]
new_year_folder = old_year_folder.replace(old_year, new_year)
new_folder = os.path.join(new_year_folder, new_movie)
else:
new_folder = old_folder.replace(old_movie, new_movie)
os.makedirs(new_folder)
# handle files
for file_name in os.listdir(old_folder):
old_file = os.path.join(old_folder, file_name)
new_file_name = file_name.replace(old_movie, new_movie)
new_file = os.path.join(new_folder, new_file_name)
os.rename(old_file, new_file)
# trash now empty folder
subprocess.call(['trash', old_folder])
def get_pending(config):
""" returns a list of movies with errors """
movie_list = get_emby_list(config)
errors_list = compare_list(movie_list)
return errors_list
def main():
""" main for fixing movie filenames """
handler = MovieNameFix()
def main(config):
""" main to lunch the id_fix """
errors_list = get_pending(config)
if not errors_list:
if not handler.pending:
print('no errors found')
sleep(2)
return
else:
skipped = rename(config, errors_list)
if skipped:
print('skipped following movies:')
for movie in skipped:
print(movie)
input('continue?')
else:
print(f'fixed {len(errors_list)} movie names.')
sleep(2)
handler.fix_errors()
sleep(2)