From cc6e0198a01595f6f529d67a485585a97e492635 Mon Sep 17 00:00:00 2001 From: simon Date: Fri, 16 Apr 2021 14:16:27 +0700 Subject: [PATCH 1/5] downloading missing trailers to archive --- interface.py | 22 ++++- src/{__init__ => __init__.py} | 0 src/trailers.py | 178 ++++++++++++++++++++++++++++++++++ 3 files changed, 197 insertions(+), 3 deletions(-) rename src/{__init__ => __init__.py} (100%) create mode 100644 src/trailers.py diff --git a/interface.py b/interface.py index 0965e78..9f88232 100755 --- a/interface.py +++ b/interface.py @@ -12,6 +12,7 @@ import src.tvsort as tvsort import src.tvsort_id as tvsort_id import src.moviesort as moviesort import src.db_export as db_export +import src.trailers as trailers def get_config(): @@ -42,6 +43,15 @@ def get_config(): config["emby_url"] = config_parser.get('emby', 'emby_url') config["emby_user_id"] = config_parser.get('emby', 'emby_user_id') config["emby_api_key"] = config_parser.get('emby', 'emby_api_key') + # youtubedl_ops + ydl_opts = dict(config_parser.items('ydl_opts')) + # dedect string literals, is there a better way to do that? + for key, value in ydl_opts.items(): + if value.isdigit(): + ydl_opts[key] = int(value) + elif value.lower() in ['true', 'false']: + ydl_opts[key] = bool(value) + config['ydl_opts'] = ydl_opts return config @@ -50,11 +60,13 @@ def get_pending_all(config): # call subfunction to collect pending pending_movie = moviesort.get_pending(config['movie_downpath']) pending_tv = tvsort.get_pending(config['tv_downpath']) - pending_total = pending_movie + pending_tv + pending_trailer = trailers.get_pending(config) + pending_total = pending_movie + pending_tv + pending_trailer # build dict pending = {} pending['movies'] = pending_movie pending['tv'] = pending_tv + pending['trailer'] = pending_trailer pending['total'] = pending_total return pending @@ -76,8 +88,10 @@ def print_menu(stdscr, current_row_idx, menu, config): pending_count = pending['movies'] elif row == 'TV shows': pending_count = pending['tv'] + elif row == 'Trailer download': + pending_count = pending['trailer'] else: - pending_count = 0 + pending_count = ' ' # center whole y = h // 2 - len(menu) + idx # print string to menu @@ -104,6 +118,8 @@ def sel_handler(menu_item, config): tvsort.main(config, tvsort_id) elif menu_item == 'DB export': db_export.main(config) + elif menu_item == 'Trailer download': + trailers.main(config) def curses_main(stdscr, menu, config): @@ -141,7 +157,7 @@ def curses_main(stdscr, menu, config): def main(): """ main wraps the curses menu """ # setup - menu = ['All', 'Movies', 'TV shows', 'DB export', 'Exit'] + menu = ['All', 'Movies', 'TV shows', 'DB export', 'Trailer download', 'Exit'] config = get_config() log_file = path.join(config["log_folder"], 'rename.log') logging.basicConfig(filename=log_file,level=logging.INFO,format='%(asctime)s:%(message)s') diff --git a/src/__init__ b/src/__init__.py similarity index 100% rename from src/__init__ rename to src/__init__.py diff --git a/src/trailers.py b/src/trailers.py new file mode 100644 index 0000000..8524f66 --- /dev/null +++ b/src/trailers.py @@ -0,0 +1,178 @@ +""" download trailers found in emby with youtube-dl """ + +import os +import re +import requests +import subprocess +import youtube_dl + +from time import sleep + + +def incomplete(config): + """ search for incomplete downloads and trash them """ + sortpath = config['sortpath'] + file_list = os.listdir(sortpath) + trashed = False + for file in file_list: + if file.endswith('.part') or file.endswith('.ytdl'): + trashed = True + file_path = os.path.join(sortpath, file) + os.path.isfile(file_path) + subprocess.call(['trash', file_path]) + return trashed + + +def get_local_trailers(config): + """ gets a list of existing trailers on filesystem """ + emby_url = config['emby_url'] + emby_api_key = config['emby_api_key'] + url = (emby_url + '/Trailers?api_key=' + emby_api_key + + '&Recursive=True&Fields=Path,MediaStreams') + r = requests.get(url).json() + local_trailer_list = [] + for movie in r['Items']: + trailer_name = movie['Name'] + trailing_pattern = re.compile(r'(.*_)([0-9a-zA-Z-_]{11})(-trailer)$') + youtube_id = trailing_pattern.findall(trailer_name)[0][1] + movie_name = movie['Path'].split('/')[-2] + media_streams = movie['MediaSources'][0]['MediaStreams'] + video_stream = list(filter(lambda stream: stream['Type'] == 'Video', media_streams)) + width = video_stream[0]['Width'] + height = video_stream[0]['Height'] + trailer_details = {'movie_name': movie_name, 'youtube_id': youtube_id, + 'width': width, 'height': height} + local_trailer_list.append(trailer_details) + return local_trailer_list + + +def get_remote_trailers(config): + """ get a list of available trailers on emby """ + emby_url = config['emby_url'] + emby_api_key = config['emby_api_key'] + # remote trailer urls + url = (emby_url + '/Items?api_key=' + emby_api_key + + '&Recursive=True&Fields=LocalTrailerCount,RemoteTrailers,Path&' + + 'IncludeItemTypes=Movie') + r = requests.get(url).json() + remote_trailers_list = [] + for movie in r['Items']: + movie_name = movie['Path'].split('/')[-2] + movie_path = '/'.join(movie['Path'].split('/')[-3:-1]) + local_trailer_count = movie['LocalTrailerCount'] + remote_trailers = movie['RemoteTrailers'] + trailer_details = {'movie_name': movie_name, 'movie_path': movie_path, + 'local_trailer_count': local_trailer_count, + 'remote_trailers': remote_trailers} + remote_trailers_list.append(trailer_details) + return remote_trailers_list + + +def compare_download(local_trailer_list, remote_trailers_list, config): + """ figure out which trailers need downloading """ + # failed before + log_file = os.path.join(config['log_folder'], 'trailers') + # check if log file exists + if not os.path.isfile(log_file): + # create empty file + open(log_file, 'a').close() + failed_ids = [] + else: + with open(log_file, 'r') as f: + lines = f.readlines() + failed_ids = [i.split()[0] for i in lines] + # ids already downloaded + local_ids = [i['youtube_id'] for i in local_trailer_list] + # find pending + pending = [] + for movie in remote_trailers_list: + movie_name = movie['movie_name'] + for trailer in movie['remote_trailers']: + vid_id = trailer['Url'].split('?v=')[1] + + if vid_id not in failed_ids and vid_id not in local_ids: + pending.append((vid_id, movie_name)) + return pending + + +def dl_pending(pending, config): + """ download pending trailers """ + sortpath = config['sortpath'] + ydl_opts = config['ydl_opts'] + # loop thrugh list + downloaded = [] + for trailer in pending: + to_down_id = trailer[0] + movie_name = trailer[1] + filename = os.path.join(sortpath, movie_name + '_' + to_down_id + '-trailer.mkv') + ydl_opts['outtmpl'] = filename + # try up to 5 times + for i in range(5): + try: + print(f'[{i}] {to_down_id} {movie_name}') + youtube_dl.YoutubeDL(ydl_opts).download(['https://www.youtube.com/watch?v=' + to_down_id]) + except KeyboardInterrupt: + return False + except: + if i == 4: + # giving up + log_file = os.path.join(config['log_folder'], 'trailers') + with open(log_file, 'a') as f: + f.write(f'{to_down_id} {movie_name}\n') + return False + else: + sleep((i + 1) ** 2) + continue + else: + downloaded.append(to_down_id) + break + return downloaded + + +def archive(config): + """ move downloaded trailers to movie archive """ + sortpath = config['sortpath'] + moviepath = config['moviepath'] + + new_trailers = os.listdir(sortpath) + # loop through new trailers + for trailer in new_trailers: + # build path + year_pattern = re.compile(r'(\()([0-9]{4})(\))') + trailing_pattern = re.compile(r'(.*)(_[0-9a-zA-Z-_]{11}-trailer\.mkv)$') + movie_name = trailing_pattern.findall(trailer)[0][0] + year = year_pattern.findall(trailer)[0][1] + movie_folder = os.path.join(moviepath, year, movie_name) + # move if all good + if os.path.isdir(movie_folder): + old_file = os.path.join(sortpath, trailer) + new_file = os.path.join(movie_folder, trailer) + os.rename(old_file, new_file) + return new_trailers + + +def get_pending(config): + """ get a list of pending trailers """ + local_trailer_list = get_local_trailers(config) + remote_trailers_list = get_remote_trailers(config) + pending = compare_download(local_trailer_list, remote_trailers_list, config) + return pending + + +def main(config): + """ main function to download pending trailers """ + # check for clean folder + trashed = incomplete(config) + # look for trailer + if not trashed: + pending = get_pending(config) + # download if needed + if pending: + print(f'downloading {len(pending)} trailers') + downloaded = dl_pending(pending, config) + else: + print('no missing trailers found') + # move to archive + if downloaded: + new_trailers = archive(config) + print(f'downloaded {len(new_trailers)} new trailers') From 31254a9e7806e5b3ab1cdb9d2012c081323badb0 Mon Sep 17 00:00:00 2001 From: simon Date: Fri, 16 Apr 2021 14:18:55 +0700 Subject: [PATCH 2/5] fixint typerror in interface pending --- interface.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/interface.py b/interface.py index 9f88232..b45e706 100755 --- a/interface.py +++ b/interface.py @@ -60,7 +60,7 @@ def get_pending_all(config): # call subfunction to collect pending pending_movie = moviesort.get_pending(config['movie_downpath']) pending_tv = tvsort.get_pending(config['tv_downpath']) - pending_trailer = trailers.get_pending(config) + pending_trailer = len(trailers.get_pending(config)) pending_total = pending_movie + pending_tv + pending_trailer # build dict pending = {} From 1648864c1e985fbbb0be412b8a1efcf05c0f2684 Mon Sep 17 00:00:00 2001 From: simon Date: Fri, 16 Apr 2021 14:42:20 +0700 Subject: [PATCH 3/5] more error handeling on failed download --- src/trailers.py | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/src/trailers.py b/src/trailers.py index 8524f66..d56b91a 100644 --- a/src/trailers.py +++ b/src/trailers.py @@ -20,6 +20,9 @@ def incomplete(config): file_path = os.path.join(sortpath, file) os.path.isfile(file_path) subprocess.call(['trash', file_path]) + if not file_list and not trashed: + new_trailers = archive(config) + print(f'moved {len(new_trailers)} into archive') return trashed @@ -119,7 +122,7 @@ def dl_pending(pending, config): log_file = os.path.join(config['log_folder'], 'trailers') with open(log_file, 'a') as f: f.write(f'{to_down_id} {movie_name}\n') - return False + break else: sleep((i + 1) ** 2) continue From ac711f6db6e5ebac064643e45df1bfe778f8a916 Mon Sep 17 00:00:00 2001 From: simon Date: Fri, 16 Apr 2021 15:42:36 +0700 Subject: [PATCH 4/5] moving get_pending call out of curses to avoid quering the db on every key press --- interface.py | 9 +++++---- 1 file changed, 5 insertions(+), 4 deletions(-) diff --git a/interface.py b/interface.py index b45e706..61be54f 100755 --- a/interface.py +++ b/interface.py @@ -71,9 +71,9 @@ def get_pending_all(config): return pending -def print_menu(stdscr, current_row_idx, menu, config): +def print_menu(stdscr, current_row_idx, menu, config, pending): """ print menu with populated pending count """ - pending = get_pending_all(config) + # build stdscr h, w = stdscr.getmaxyx() longest = len(max(menu)) @@ -127,7 +127,8 @@ def curses_main(stdscr, menu, config): curses.curs_set(0) curses.init_pair(1, curses.COLOR_BLUE, curses.COLOR_WHITE) current_row_idx = 0 - print_menu(stdscr, current_row_idx, menu, config) + pending = get_pending_all(config) + print_menu(stdscr, current_row_idx, menu, config, pending) # endless loop while True: # wait for exit signal @@ -147,7 +148,7 @@ def curses_main(stdscr, menu, config): # exit curses and do something return menu_item # print - print_menu(stdscr, current_row_idx, menu, config) + print_menu(stdscr, current_row_idx, menu, config, pending) stdscr.refresh() except KeyboardInterrupt: # clean exit on ctrl + c From 03510b183eaaf9bc397bd81e2d27026f1a23d314 Mon Sep 17 00:00:00 2001 From: simon Date: Fri, 16 Apr 2021 15:45:19 +0700 Subject: [PATCH 5/5] update trailer dl documentation and requirements --- README.md | 21 ++++++++++++++++++--- config.sample | 8 +++++++- requirements.txt | 2 ++ src/trailers.py | 1 + 4 files changed, 28 insertions(+), 4 deletions(-) diff --git a/README.md b/README.md index 813b38b..2deb027 100644 --- a/README.md +++ b/README.md @@ -23,12 +23,23 @@ Episodes are named in this style, a more flexible solution is in pending: ## db_export Export the library to csv files. Calles the Emby API to get a list of movies and episodes and exports this to a convenient set ov CSV files. +## trailers +Downloading trailers from links provided from emby and move them into the movie folder. +Trailers are named in this style, a more flexible solution is in pending: +**{movie-name} {Year}_{youtube-id}_trailer.mkv** + ## setup ### install These are the none standard Python libraries in use in this project: * [requests](https://pypi.org/project/requests/) * Install on Arch: `sudo pacman -Qi python-request` - * Install with pip `pip install request` + * Install with pip: `pip install request` +* [trash-cli](https://pypi.org/project/trash-cli/) + * Install on Arch: `sudo pacman -S trash-cli` + * Install with pip: `pip install trash-cli` +* [youtube-dl](https://pypi.org/project/youtube_dl/) + * Install on Arch: `sudo pacman -S youtube-dl` + * Install with pip: `pip install youtube_d` * curses * Is already installed on most linux based systems. * On Windows: `pip install windows-curses` @@ -43,7 +54,11 @@ Duplicate the config.sample file to a file named *config* and set the following * `ext`: A space separated list of valid media file extensions to easily filter out none media related files. * `log_path`: Path to a folder to output all renaming done to keep track and check for any errors and safe csv files. * `movie_db_api`: Register and get your themoviedb.com **API Key (v3 auth)** acces from [here](https://www.themoviedb.org/settings/api). -Emby integration: + +*Emby integration:* * `emby_url`: url where your emby instance is reachable * `emby_user_id`: user id of your emby user -* `emby_api_key`: api key for your user on emby +* `emby_api_key`: api key for your user on emby + +*Trailer download:* +Arguments under the [ydl_opts] section will get passed in to youtube-dl. Check out the documentation for details. \ No newline at end of file diff --git a/config.sample b/config.sample index 4967691..094c968 100644 --- a/config.sample +++ b/config.sample @@ -11,4 +11,10 @@ movie_db_api = aaaabbbbccccdddd1111222233333444 [emby] emby_url = http://media.local:8096/emby emby_user_id = aaaa1111bbbb2222cccc3333dddd4444 -emby_api_key = eeee5555ffff6666gggg7777hhhh8888 \ No newline at end of file +emby_api_key = eeee5555ffff6666gggg7777hhhh8888 + +[ydl_opts] +format = bestvideo[height<=1080]+bestaudio/best[height<=1080] +merge_output_format = mkv +external_downloader = aria2c +geo_bypass = True diff --git a/requirements.txt b/requirements.txt index 3afc717..cb8f841 100644 --- a/requirements.txt +++ b/requirements.txt @@ -1 +1,3 @@ request +trash-cli +youtube_dl diff --git a/src/trailers.py b/src/trailers.py index d56b91a..e342449 100644 --- a/src/trailers.py +++ b/src/trailers.py @@ -179,3 +179,4 @@ def main(config): if downloaded: new_trailers = archive(config) print(f'downloaded {len(new_trailers)} new trailers') + sleep(2)