From 6985c4da415102783478387c26e967cd60344ca5 Mon Sep 17 00:00:00 2001 From: Drugsosos <44712637+Drugsosos@users.noreply.github.com> Date: Thu, 9 Jun 2022 02:01:12 +0300 Subject: [PATCH 001/126] added name_normalize to fix bugs with forbidden chars in videoname --- video_creation/final_video.py | 11 ++++++++++- 1 file changed, 10 insertions(+), 1 deletion(-) diff --git a/video_creation/final_video.py b/video_creation/final_video.py index a12f2d6..07b0ffc 100644 --- a/video_creation/final_video.py +++ b/video_creation/final_video.py @@ -17,6 +17,15 @@ import os W, H = 1080, 1920 +def name_normalize( + name: str +) -> str: + name = re.sub(r'[?\\"%*:|<>]', '', name) + name = re.sub(r'(\D+)/(\D+)', r'\1\ or\ \2', name) + name = re.sub(r'(\d+)/(\d+)', r'\1\ of\ \2', name) + return name + + def make_final_video(number_of_clips): # Calls opacity from the .env @@ -83,7 +92,7 @@ def make_final_video(number_of_clips): final_video_path = "assets/" if os.getenv("FINAL_VIDEO_PATH"): final_video_path = os.getenv("FINAL_VIDEO_PATH") - filename = (re.sub('[?\"%*:|<>]', '', (final_video_path + reddit.subreddit.submission.title + ".mp4"))) + filename = final_video_path + name_normalize(reddit.subreddit.submission.title) + ".mp4" try: final.write_videofile(filename, fps=30, audio_codec="aac", audio_bitrate="192k") except: From 1764d54e5ddbfb6c42dbd0b0e733ce532da9f8f3 Mon Sep 17 00:00:00 2001 From: Drugsosos <44712637+Drugsosos@users.noreply.github.com> Date: Thu, 9 Jun 2022 02:01:12 +0300 Subject: [PATCH 002/126] added name_normalize to fix bugs with forbidden chars in videoname --- video_creation/final_video.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/video_creation/final_video.py b/video_creation/final_video.py index 07b0ffc..27a351d 100644 --- a/video_creation/final_video.py +++ b/video_creation/final_video.py @@ -21,8 +21,8 @@ def name_normalize( name: str ) -> str: name = re.sub(r'[?\\"%*:|<>]', '', name) - name = re.sub(r'(\D+)/(\D+)', r'\1\ or\ \2', name) - name = re.sub(r'(\d+)/(\d+)', r'\1\ of\ \2', name) + name = re.sub(r'(\D+)/(\D+)', r'\1 or\ \2', name) + name = re.sub(r'(\d+)/(\d+)', r'\1 of\ \2', name) return name From 30ad5a510d9767f4d08821f0c45dc6b2352ecb67 Mon Sep 17 00:00:00 2001 From: Drugsosos <44712637+Drugsosos@users.noreply.github.com> Date: Fri, 10 Jun 2022 20:17:34 +0300 Subject: [PATCH 003/126] fixes in regex in name_normalize --- video_creation/final_video.py | 7 +++++-- 1 file changed, 5 insertions(+), 2 deletions(-) diff --git a/video_creation/final_video.py b/video_creation/final_video.py index 27a351d..f2acf74 100644 --- a/video_creation/final_video.py +++ b/video_creation/final_video.py @@ -21,8 +21,11 @@ def name_normalize( name: str ) -> str: name = re.sub(r'[?\\"%*:|<>]', '', name) - name = re.sub(r'(\D+)/(\D+)', r'\1 or\ \2', name) - name = re.sub(r'(\d+)/(\d+)', r'\1 of\ \2', name) + name = re.sub(r'( [w,W]\s?\/\s?[o,O,0])', r' without', name) + name = re.sub(r'( [w,W]\s?\/)', r' with', name) + name = re.sub(r'([0-9]+)\s?\/\s?([0-9]+)', r'\1 of \2', name) + name = re.sub(r'(\w+)\s?\/\s?(\w+)', r'\1 or \2', name) + name = re.sub(r'\/', r'', name) return name From ce6a87aac2a2381e0c83d1d9999940359060cf13 Mon Sep 17 00:00:00 2001 From: Jason Date: Fri, 1 Jul 2022 21:17:07 -0400 Subject: [PATCH 004/126] fix: Fixed Mutagen error issue style: made final_video.py cleaner --- TTS/engine_wrapper.py | 9 +++++++-- video_creation/final_video.py | 4 +--- 2 files changed, 8 insertions(+), 5 deletions(-) diff --git a/TTS/engine_wrapper.py b/TTS/engine_wrapper.py index bbe4e9a..3e69d54 100644 --- a/TTS/engine_wrapper.py +++ b/TTS/engine_wrapper.py @@ -3,6 +3,9 @@ from pathlib import Path from typing import Tuple import re from os import getenv + +import sox +from mutagen import MutagenError from mutagen.mp3 import MP3 import translators as ts from rich.progress import track @@ -91,8 +94,10 @@ class TTSEngine: def call_tts(self, filename: str, text: str): self.tts_module.run(text=process_text(text), filepath=f"{self.path}/{filename}.mp3") - self.length += MP3(f"{self.path}/{filename}.mp3").info.length - + try: + self.length += MP3(f"{self.path}/{filename}.mp3").info.length + except MutagenError: + self.length += sox.file_info.duration(f"{self.path}/{filename}.mp3") def process_text(text: str): lang = getenv("POSTLANG", "") diff --git a/video_creation/final_video.py b/video_creation/final_video.py index d170169..f31d68c 100755 --- a/video_creation/final_video.py +++ b/video_creation/final_video.py @@ -44,9 +44,7 @@ def make_final_video(number_of_clips: int, length: int, reddit_obj: dict[str]): ) # Gather all audio clips - audio_clips = [] - for i in range(0, number_of_clips): - audio_clips.append(AudioFileClip(f"assets/temp/mp3/{i}.mp3")) + audio_clips = [AudioFileClip(f"assets/mp3/{i}.mp3") for i in range(number_of_clips)] audio_clips.insert(0, AudioFileClip("assets/temp/mp3/title.mp3")) audio_concat = concatenate_audioclips(audio_clips) audio_composite = CompositeAudioClip([audio_concat]) From e950d47ddc402b11989ce323ed1cc36cfaddc48d Mon Sep 17 00:00:00 2001 From: Jason Date: Fri, 1 Jul 2022 21:27:10 -0400 Subject: [PATCH 005/126] forgot to update vers --- TTS/engine_wrapper.py | 4 ++-- main.py | 2 +- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/TTS/engine_wrapper.py b/TTS/engine_wrapper.py index 3e69d54..a06b4b9 100644 --- a/TTS/engine_wrapper.py +++ b/TTS/engine_wrapper.py @@ -6,7 +6,7 @@ from os import getenv import sox from mutagen import MutagenError -from mutagen.mp3 import MP3 +from mutagen.mp3 import MP3, HeaderNotFoundError import translators as ts from rich.progress import track from moviepy.editor import AudioFileClip, CompositeAudioClip, concatenate_audioclips @@ -96,7 +96,7 @@ class TTSEngine: self.tts_module.run(text=process_text(text), filepath=f"{self.path}/{filename}.mp3") try: self.length += MP3(f"{self.path}/{filename}.mp3").info.length - except MutagenError: + except (MutagenError, HeaderNotFoundError): self.length += sox.file_info.duration(f"{self.path}/{filename}.mp3") def process_text(text: str): diff --git a/main.py b/main.py index c85a2aa..81bec26 100755 --- a/main.py +++ b/main.py @@ -14,7 +14,7 @@ from video_creation.final_video import make_final_video from video_creation.screenshot_downloader import download_screenshots_of_reddit_posts from video_creation.voices import save_text_to_mp3 -VERSION = "2.2.1" +VERSION = "2.2.2" print( """ ██████╗ ███████╗██████╗ ██████╗ ██╗████████╗ ██╗ ██╗██╗██████╗ ███████╗ ██████╗ ███╗ ███╗ █████╗ ██╗ ██╗███████╗██████╗ From 50392aac60f2b8478a4e753ae0682d453c19fe21 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Muhammed=20Mustafa=20Ak=C5=9Fam?= Date: Sat, 2 Jul 2022 12:41:36 +0300 Subject: [PATCH 006/126] Update final_video.py fixed legacy directory issue --- video_creation/final_video.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/video_creation/final_video.py b/video_creation/final_video.py index f31d68c..852c4b2 100755 --- a/video_creation/final_video.py +++ b/video_creation/final_video.py @@ -44,7 +44,7 @@ def make_final_video(number_of_clips: int, length: int, reddit_obj: dict[str]): ) # Gather all audio clips - audio_clips = [AudioFileClip(f"assets/mp3/{i}.mp3") for i in range(number_of_clips)] + audio_clips = [AudioFileClip(f"assets/temp/mp3/{i}.mp3") for i in range(number_of_clips)] audio_clips.insert(0, AudioFileClip("assets/temp/mp3/title.mp3")) audio_concat = concatenate_audioclips(audio_clips) audio_composite = CompositeAudioClip([audio_concat]) From 31b09dca38193af5f9217cabdec90e14c3b1ba74 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Muhammed=20Mustafa=20Ak=C5=9Fam?= Date: Sun, 3 Jul 2022 12:20:14 +0300 Subject: [PATCH 007/126] Update .env.template updated .env.template to match the regex --- .env.template | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.env.template b/.env.template index 7cf7cc2..77f2acf 100644 --- a/.env.template +++ b/.env.template @@ -14,7 +14,7 @@ REDDIT_CLIENT_SECRET="" #fFAGRNJru1FTz70BzhT3Zg REDDIT_USERNAME="" #asdfghjkl #EXPLANATION the username of your reddit account #RANGE 3:20 -#MATCH_REGEX [_0-9a-zA-Z]+$ +#MATCH_REGEX [-_0-9a-zA-Z]+$ #OOB_ERROR A username HAS to be between 3 and 20 characters REDDIT_PASSWORD="" #fFAGRNJru1FTz70BzhT3Zg From 708016c5839250dc064d65708a57181c3ef0aec6 Mon Sep 17 00:00:00 2001 From: Callum Leslie Date: Sun, 3 Jul 2022 17:42:18 +0100 Subject: [PATCH 008/126] fix: mutagen error - moved to moviepy duration --- TTS/engine_wrapper.py | 11 ++++++----- 1 file changed, 6 insertions(+), 5 deletions(-) diff --git a/TTS/engine_wrapper.py b/TTS/engine_wrapper.py index a06b4b9..692cfdc 100644 --- a/TTS/engine_wrapper.py +++ b/TTS/engine_wrapper.py @@ -94,11 +94,12 @@ class TTSEngine: def call_tts(self, filename: str, text: str): self.tts_module.run(text=process_text(text), filepath=f"{self.path}/{filename}.mp3") - try: - self.length += MP3(f"{self.path}/{filename}.mp3").info.length - except (MutagenError, HeaderNotFoundError): - self.length += sox.file_info.duration(f"{self.path}/{filename}.mp3") - + # try: + # self.length += MP3(f"{self.path}/{filename}.mp3").info.length + # except (MutagenError, HeaderNotFoundError): + # self.length += sox.file_info.duration(f"{self.path}/{filename}.mp3") + clip = AudioFileClip(f"{self.path}/{filename}.mp3") + self.length += clip.duration def process_text(text: str): lang = getenv("POSTLANG", "") new_text = sanitize_text(text) From 90be9e09b3dc5027f11ebdd83e6a149f5f256e80 Mon Sep 17 00:00:00 2001 From: Jason Date: Sun, 3 Jul 2022 12:43:44 -0400 Subject: [PATCH 009/126] fixed the typehiting that @HallowedDust5 broke for py versions < 3.9 --- utils/videos.py | 7 ++++--- video_creation/final_video.py | 4 +++- video_creation/screenshot_downloader.py | 5 +++-- video_creation/voices.py | 3 ++- 4 files changed, 12 insertions(+), 7 deletions(-) diff --git a/utils/videos.py b/utils/videos.py index f675ecf..3a456cd 100755 --- a/utils/videos.py +++ b/utils/videos.py @@ -2,6 +2,7 @@ import json import os import time from os import getenv +from typing import Dict from praw.models import Submission @@ -9,16 +10,16 @@ from utils.console import print_step def check_done( - redditobj: dict[str], + redditobj: Dict[str], ) -> Submission: # don't set this to be run anyplace that isn't subreddit.py bc of inspect stack """Checks if the chosen post has already been generated Args: - redditobj (dict[str]): Reddit object gotten from reddit/subreddit.py + redditobj (Dict[str]): Reddit object gotten from reddit/subreddit.py Returns: - dict[str]|None: Reddit object in args + Dict[str]|None: Reddit object in args """ with open("./video_creation/data/videos.json", "r", encoding="utf-8") as done_vids_raw: diff --git a/video_creation/final_video.py b/video_creation/final_video.py index 852c4b2..e429dbc 100755 --- a/video_creation/final_video.py +++ b/video_creation/final_video.py @@ -3,6 +3,7 @@ import multiprocessing import os import re from os.path import exists +from typing import Dict from moviepy.editor import ( VideoFileClip, @@ -25,12 +26,13 @@ console = Console() W, H = 1080, 1920 -def make_final_video(number_of_clips: int, length: int, reddit_obj: dict[str]): +def make_final_video(number_of_clips: int, length: int, reddit_obj: Dict[str]): """Gathers audio clips, gathers all screenshots, stitches them together and saves the final video to assets/temp Args: number_of_clips (int): Index to end at when going through the screenshots length (int): Length of the video + reddit_obj (Dict[str]): The reddit object that contains the posts to read. """ print_step("Creating the final video 🎥") VideoFileClip.reW = lambda clip: clip.resize(width=W) diff --git a/video_creation/screenshot_downloader.py b/video_creation/screenshot_downloader.py index aa1c9d9..670a472 100644 --- a/video_creation/screenshot_downloader.py +++ b/video_creation/screenshot_downloader.py @@ -2,6 +2,7 @@ import json import os from os import getenv from pathlib import Path +from typing import Dict from playwright.async_api import async_playwright # pylint: disable=unused-import @@ -16,11 +17,11 @@ from utils.console import print_step, print_substep storymode = False -def download_screenshots_of_reddit_posts(reddit_object: dict[str], screenshot_num: int): +def download_screenshots_of_reddit_posts(reddit_object: Dict[str], screenshot_num: int): """Downloads screenshots of reddit posts as seen on the web. Downloads to assets/temp/png Args: - reddit_object (dict[str]): Reddit object received from reddit/subreddit.py + reddit_object (Dict[str]): Reddit object received from reddit/subreddit.py screenshot_num (int): Number of screenshots to downlaod """ diff --git a/video_creation/voices.py b/video_creation/voices.py index 240c851..d02a14e 100644 --- a/video_creation/voices.py +++ b/video_creation/voices.py @@ -1,6 +1,7 @@ #!/usr/bin/env python import os +from typing import Dict, Tuple from rich.console import Console @@ -23,7 +24,7 @@ TTSProviders = { } -def save_text_to_mp3(reddit_obj: dict[str]) -> tuple[int, int]: +def save_text_to_mp3(reddit_obj: Dict[str]) -> Tuple[int, int]: """Saves text to MP3 files. Args: From 65e42f286d5505f1af42d69612e8909c7cf418e9 Mon Sep 17 00:00:00 2001 From: Jason Date: Sun, 3 Jul 2022 12:52:51 -0400 Subject: [PATCH 010/126] style: improved typehinting --- utils/videos.py | 3 +-- video_creation/final_video.py | 4 ++-- video_creation/screenshot_downloader.py | 3 +-- video_creation/voices.py | 2 +- 4 files changed, 5 insertions(+), 7 deletions(-) diff --git a/utils/videos.py b/utils/videos.py index 3a456cd..07659f6 100755 --- a/utils/videos.py +++ b/utils/videos.py @@ -10,7 +10,7 @@ from utils.console import print_step def check_done( - redditobj: Dict[str], + redditobj: Submission, ) -> Submission: # don't set this to be run anyplace that isn't subreddit.py bc of inspect stack """Checks if the chosen post has already been generated @@ -21,7 +21,6 @@ def check_done( Returns: Dict[str]|None: Reddit object in args """ - with open("./video_creation/data/videos.json", "r", encoding="utf-8") as done_vids_raw: done_videos = json.load(done_vids_raw) for video in done_videos: diff --git a/video_creation/final_video.py b/video_creation/final_video.py index e429dbc..81b9b98 100755 --- a/video_creation/final_video.py +++ b/video_creation/final_video.py @@ -26,13 +26,13 @@ console = Console() W, H = 1080, 1920 -def make_final_video(number_of_clips: int, length: int, reddit_obj: Dict[str]): +def make_final_video(number_of_clips: int, length: int, reddit_obj: dict): """Gathers audio clips, gathers all screenshots, stitches them together and saves the final video to assets/temp Args: number_of_clips (int): Index to end at when going through the screenshots length (int): Length of the video - reddit_obj (Dict[str]): The reddit object that contains the posts to read. + reddit_obj (dict): The reddit object that contains the posts to read. """ print_step("Creating the final video 🎥") VideoFileClip.reW = lambda clip: clip.resize(width=W) diff --git a/video_creation/screenshot_downloader.py b/video_creation/screenshot_downloader.py index 670a472..e8afc44 100644 --- a/video_creation/screenshot_downloader.py +++ b/video_creation/screenshot_downloader.py @@ -17,14 +17,13 @@ from utils.console import print_step, print_substep storymode = False -def download_screenshots_of_reddit_posts(reddit_object: Dict[str], screenshot_num: int): +def download_screenshots_of_reddit_posts(reddit_object: dict, screenshot_num: int): """Downloads screenshots of reddit posts as seen on the web. Downloads to assets/temp/png Args: reddit_object (Dict[str]): Reddit object received from reddit/subreddit.py screenshot_num (int): Number of screenshots to downlaod """ - print_step("Downloading screenshots of reddit posts...") # ! Make sure the reddit screenshots folder exists diff --git a/video_creation/voices.py b/video_creation/voices.py index d02a14e..5105a10 100644 --- a/video_creation/voices.py +++ b/video_creation/voices.py @@ -24,7 +24,7 @@ TTSProviders = { } -def save_text_to_mp3(reddit_obj: Dict[str]) -> Tuple[int, int]: +def save_text_to_mp3(reddit_obj) -> Tuple[int, int]: """Saves text to MP3 files. Args: From 1c883e5ccd44771c42ad87b1fc87c65ead652079 Mon Sep 17 00:00:00 2001 From: Jason Date: Sun, 3 Jul 2022 12:55:27 -0400 Subject: [PATCH 011/126] fix: maybe fixed mutagen error using callums method --- TTS/engine_wrapper.py | 10 +++++++--- 1 file changed, 7 insertions(+), 3 deletions(-) diff --git a/TTS/engine_wrapper.py b/TTS/engine_wrapper.py index a06b4b9..ffabd35 100644 --- a/TTS/engine_wrapper.py +++ b/TTS/engine_wrapper.py @@ -95,9 +95,13 @@ class TTSEngine: def call_tts(self, filename: str, text: str): self.tts_module.run(text=process_text(text), filepath=f"{self.path}/{filename}.mp3") try: - self.length += MP3(f"{self.path}/{filename}.mp3").info.length - except (MutagenError, HeaderNotFoundError): - self.length += sox.file_info.duration(f"{self.path}/{filename}.mp3") + try: + self.length += MP3(f"{self.path}/{filename}.mp3").info.length + except (MutagenError, HeaderNotFoundError): + self.length += sox.file_info.duration(f"{self.path}/{filename}.mp3") + except Exception: # last resort backup + clip = AudioFileClip(f"{self.path}/{filename}.mp3") + self.length += clip.duration def process_text(text: str): lang = getenv("POSTLANG", "") From edf981ac399763a5e1d34ca5642bc7af5c9bedd5 Mon Sep 17 00:00:00 2001 From: Callum Leslie Date: Sun, 3 Jul 2022 18:12:11 +0100 Subject: [PATCH 012/126] chore: reduce code duplication --- video_creation/final_video.py | 47 ++++++++++++----------------------- 1 file changed, 16 insertions(+), 31 deletions(-) diff --git a/video_creation/final_video.py b/video_creation/final_video.py index 81b9b98..23876ed 100755 --- a/video_creation/final_video.py +++ b/video_creation/final_video.py @@ -61,41 +61,26 @@ def make_final_video(number_of_clips: int, length: int, reddit_obj: dict): # add title to video image_clips = [] # Gather all images - if opacity is None or float(opacity) >= 1: # opacity not set or is set to one OR MORE - image_clips.insert( - 0, - ImageClip("assets/temp/png/title.png") - .set_duration(audio_clips[0].duration) - .set_position("center") - .resize(width=W - 100), - ) - else: - image_clips.insert( - 0, - ImageClip("assets/temp/png/title.png") - .set_duration(audio_clips[0].duration) + new_opacity = 1 if opacity is None or float(opacity) >= 1 else opacity + + image_clips.insert( + 0, + ImageClip("assets/temp/png/title.png") + .set_duration(audio_clips[0].duration) + .set_position("center") + .resize(width=W - 100) + .set_opacity(new_opacity) + ) + + for i in range(0, number_of_clips): + image_clips.append( + ImageClip(f"assets/temp/png/comment_{i}.png") + .set_duration(audio_clips[i + 1].duration) .set_position("center") .resize(width=W - 100) - .set_opacity(float(opacity)), + .set_opacity(new_opacity) ) - for i in range(0, number_of_clips): - if opacity is None or float(opacity) >= 1: # opacity not set or is set to one OR MORE - image_clips.append( - ImageClip(f"assets/temp/png/comment_{i}.png") - .set_duration(audio_clips[i + 1].duration) - .set_position("center") - .resize(width=W - 100), - ) - else: - image_clips.append( - ImageClip(f"assets/temp/png/comment_{i}.png") - .set_duration(audio_clips[i + 1].duration) - .set_position("center") - .resize(width=W - 100) - .set_opacity(float(opacity)), - ) - # if os.path.exists("assets/mp3/posttext.mp3"): # image_clips.insert( # 0, From 11de0fd3240aeff30d4e0ccd9a4ef735c20a98d8 Mon Sep 17 00:00:00 2001 From: Callum Leslie Date: Sun, 3 Jul 2022 18:47:35 +0100 Subject: [PATCH 013/126] fix: length checker works with new duration code --- TTS/engine_wrapper.py | 7 ++++--- main.py | 3 ++- video_creation/final_video.py | 11 ++--------- 3 files changed, 8 insertions(+), 13 deletions(-) diff --git a/TTS/engine_wrapper.py b/TTS/engine_wrapper.py index 692cfdc..77f6eab 100644 --- a/TTS/engine_wrapper.py +++ b/TTS/engine_wrapper.py @@ -4,9 +4,9 @@ from typing import Tuple import re from os import getenv -import sox -from mutagen import MutagenError -from mutagen.mp3 import MP3, HeaderNotFoundError +# import sox +# from mutagen import MutagenError +# from mutagen.mp3 import MP3, HeaderNotFoundError import translators as ts from rich.progress import track from moviepy.editor import AudioFileClip, CompositeAudioClip, concatenate_audioclips @@ -100,6 +100,7 @@ class TTSEngine: # self.length += sox.file_info.duration(f"{self.path}/{filename}.mp3") clip = AudioFileClip(f"{self.path}/{filename}.mp3") self.length += clip.duration + def process_text(text: str): lang = getenv("POSTLANG", "") new_text = sanitize_text(text) diff --git a/main.py b/main.py index 81bec26..e362aad 100755 --- a/main.py +++ b/main.py @@ -1,5 +1,5 @@ #!/usr/bin/env python - +import math from subprocess import Popen from os import getenv, name from dotenv import load_dotenv @@ -35,6 +35,7 @@ def main(POST_ID=None): cleanup() reddit_object = get_subreddit_threads(POST_ID) length, number_of_comments = save_text_to_mp3(reddit_object) + length = math.ceil(length) download_screenshots_of_reddit_posts(reddit_object, number_of_comments) download_background() chop_background_video(length) diff --git a/video_creation/final_video.py b/video_creation/final_video.py index 852c4b2..418254c 100755 --- a/video_creation/final_video.py +++ b/video_creation/final_video.py @@ -3,7 +3,6 @@ import multiprocessing import os import re from os.path import exists - from moviepy.editor import ( VideoFileClip, AudioFileClip, @@ -49,13 +48,7 @@ def make_final_video(number_of_clips: int, length: int, reddit_obj: dict[str]): audio_concat = concatenate_audioclips(audio_clips) audio_composite = CompositeAudioClip([audio_concat]) - # Get sum of all clip lengths - total_length = sum([clip.duration for clip in audio_clips]) - # round total_length to an integer - int_total_length = round(total_length) - # Output Length - - console.log(f"[bold green] Video Will Be: {int_total_length} Seconds Long") + console.log(f"[bold green] Video Will Be: {length} Seconds Long") # add title to video image_clips = [] # Gather all images @@ -127,7 +120,7 @@ def make_final_video(number_of_clips: int, length: int, reddit_obj: dict[str]): threads=multiprocessing.cpu_count(), ) ffmpeg_tools.ffmpeg_extract_subclip( - "assets/temp/temp.mp4", 0, length, targetname=f"results/{subreddit}/{filename}" + "assets/temp/temp.mp4", 0, final.duration, targetname=f"results/{subreddit}/{filename}" ) # os.remove("assets/temp/temp.mp4") From 6412aa546b7cb3f87da0408f95c60306d66be171 Mon Sep 17 00:00:00 2001 From: Callum Leslie Date: Mon, 4 Jul 2022 00:07:17 +0100 Subject: [PATCH 014/126] fix: add float cast to opacity to fix bug --- video_creation/final_video.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/video_creation/final_video.py b/video_creation/final_video.py index 23876ed..3e1f52e 100755 --- a/video_creation/final_video.py +++ b/video_creation/final_video.py @@ -61,7 +61,7 @@ def make_final_video(number_of_clips: int, length: int, reddit_obj: dict): # add title to video image_clips = [] # Gather all images - new_opacity = 1 if opacity is None or float(opacity) >= 1 else opacity + new_opacity = 1 if opacity is None or float(opacity) >= 1 else float(opacity) image_clips.insert( 0, From ada9b692993df9210622312813b48c2d2500ec2a Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Muhammed=20Mustafa=20Ak=C5=9Fam?= Date: Mon, 4 Jul 2022 11:06:26 +0300 Subject: [PATCH 015/126] Fixing legacy VOICE variables Fixing legacy VOICE variables that caused lot of issues if TTSCHOICE set to AWS_VOICE or STREAMLABS_VOICE --- TTS/aws_polly.py | 4 ++-- TTS/streamlabs_polly.py | 4 ++-- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/TTS/aws_polly.py b/TTS/aws_polly.py index 703aa6a..6cbe4c5 100644 --- a/TTS/aws_polly.py +++ b/TTS/aws_polly.py @@ -35,9 +35,9 @@ class AWSPolly: if random_voice: voice = self.randomvoice() else: - if not os.getenv("VOICE"): + if not os.getenv("AWS_VOICE"): return ValueError( - f"Please set the environment variable VOICE to a valid voice. options are: {voices}" + f"Please set the environment variable AWS_VOICE to a valid voice. options are: {voices}" ) voice = str(os.getenv("AWS_VOICE")).capitalize() try: diff --git a/TTS/streamlabs_polly.py b/TTS/streamlabs_polly.py index 41fe269..066fa53 100644 --- a/TTS/streamlabs_polly.py +++ b/TTS/streamlabs_polly.py @@ -35,9 +35,9 @@ class StreamlabsPolly: if random_voice: voice = self.randomvoice() else: - if not os.getenv("VOICE"): + if not os.getenv("STREAMLABS_VOICE"): return ValueError( - f"Please set the environment variable VOICE to a valid voice. options are: {voices}" + f"Please set the environment variable STREAMLABS_VOICE to a valid voice. options are: {voices}" ) voice = str(os.getenv("STREAMLABS_VOICE")).capitalize() body = {"voice": voice, "text": text, "service": "polly"} From b8e9ce073a3d0ff061beb90934b750f5ffe73c04 Mon Sep 17 00:00:00 2001 From: Jason Date: Mon, 4 Jul 2022 13:29:55 -0400 Subject: [PATCH 016/126] docs: changed default voice --- .env.template | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.env.template b/.env.template index 7cf7cc2..563cd07 100644 --- a/.env.template +++ b/.env.template @@ -68,7 +68,7 @@ OPACITY="1" #.8 POSTLANG="" #EXPLANATION Activates the translation feature, set the language code for translate or leave blank -TTSCHOICE="Polly" +TTSCHOICE="StreamlabsPolly" #EXPLANATION the backend used for TTS. Without anything specified, the user will be prompted to choose one. # IMPORTANT NOTE: if you use translate, you need to set this to googletranslate or tiktok and use custom voice in your language From a335e67c8cec393447067db79e5b88def6078a6a Mon Sep 17 00:00:00 2001 From: Callum Leslie Date: Mon, 4 Jul 2022 19:10:22 +0100 Subject: [PATCH 017/126] fix: stop broken typechecking --- TTS/engine_wrapper.py | 2 +- video_creation/background.py | 3 ++- 2 files changed, 3 insertions(+), 2 deletions(-) diff --git a/TTS/engine_wrapper.py b/TTS/engine_wrapper.py index 77f6eab..fd2d36b 100644 --- a/TTS/engine_wrapper.py +++ b/TTS/engine_wrapper.py @@ -72,7 +72,7 @@ class TTSEngine: print_substep("Saved Text to MP3 files successfully.", style="bold green") return self.length, idx - def split_post(self, text: str, idx: int) -> str: + def split_post(self, text: str, idx: int): split_files = [] split_text = [ x.group().strip() diff --git a/video_creation/background.py b/video_creation/background.py index 7bf6ae2..2654499 100644 --- a/video_creation/background.py +++ b/video_creation/background.py @@ -2,6 +2,7 @@ import random from os import listdir, environ from pathlib import Path from random import randrange +from typing import Tuple from moviepy.editor import VideoFileClip from moviepy.video.io.ffmpeg_tools import ffmpeg_extract_subclip @@ -10,7 +11,7 @@ from pytube import YouTube from utils.console import print_step, print_substep -def get_start_and_end_times(video_length: int, length_of_clip: int) -> tuple[int, int]: +def get_start_and_end_times(video_length: int, length_of_clip: int) -> Tuple[int, int]: """Generates a random interval of time to be used as the background of the video. Args: From 8365121865d995929df86576eaa75c6d8edd329f Mon Sep 17 00:00:00 2001 From: Callum Leslie Date: Mon, 4 Jul 2022 19:59:33 +0100 Subject: [PATCH 018/126] fix: close clip once finished --- TTS/engine_wrapper.py | 1 + 1 file changed, 1 insertion(+) diff --git a/TTS/engine_wrapper.py b/TTS/engine_wrapper.py index fd2d36b..017cf79 100644 --- a/TTS/engine_wrapper.py +++ b/TTS/engine_wrapper.py @@ -100,6 +100,7 @@ class TTSEngine: # self.length += sox.file_info.duration(f"{self.path}/{filename}.mp3") clip = AudioFileClip(f"{self.path}/{filename}.mp3") self.length += clip.duration + clip.close() def process_text(text: str): lang = getenv("POSTLANG", "") From c4da2fa55a4d30e28838a7aa58e0f08f0dc8ea38 Mon Sep 17 00:00:00 2001 From: Callum Leslie Date: Mon, 4 Jul 2022 20:33:59 +0100 Subject: [PATCH 019/126] fix: close ALL clips when finished --- TTS/engine_wrapper.py | 10 ++++++++-- 1 file changed, 8 insertions(+), 2 deletions(-) diff --git a/TTS/engine_wrapper.py b/TTS/engine_wrapper.py index 017cf79..13ff850 100644 --- a/TTS/engine_wrapper.py +++ b/TTS/engine_wrapper.py @@ -88,9 +88,15 @@ class TTSEngine: f"{self.path}/{idx}.mp3", fps=44100, verbose=False, logger=None ) - for i in range(0, idy + 1): + for i in split_files: + name = i.filename + i.close() + Path(name).unlink() + + # for i in range(0, idy + 1): # print(f"Cleaning up {self.path}/{idx}-{i}.part.mp3") - Path(f"{self.path}/{idx}-{i}.part.mp3").unlink() + + # Path(f"{self.path}/{idx}-{i}.part.mp3").unlink() def call_tts(self, filename: str, text: str): self.tts_module.run(text=process_text(text), filepath=f"{self.path}/{filename}.mp3") From 421502b93b837556d593e0810abdc6d7eb3e29d3 Mon Sep 17 00:00:00 2001 From: William9923 Date: Wed, 29 Jun 2022 21:32:21 +0700 Subject: [PATCH 020/126] feat: screenshot position config & add download progress bar --- .env.template | 5 ++ main.py | 9 +-- video_creation/background.py | 109 ++++++++++++++++++++++++---------- video_creation/final_video.py | 51 ++++++++++------ 4 files changed, 121 insertions(+), 53 deletions(-) diff --git a/.env.template b/.env.template index 77f2acf..ce28209 100644 --- a/.env.template +++ b/.env.template @@ -81,6 +81,11 @@ AWS_VOICE="Joanna" TIKTOK_VOICE="en_us_006" #EXPLANATION Sets the voice for the TikTok TTS Engine. Check the file for more information on different voices. +#OPTIONAL +BackgroundChoice="minecraft" +#EXPLANATION Sets the background for the video. Current available option : (minecraft,gta,rocket-league,motor-gta), but you can add other video easily (1080p). Check the file for more information on different background. + #OPTIONAL STORYMODE="False" # IN-PROGRESS - not yet implemented + diff --git a/main.py b/main.py index e362aad..8d134a7 100755 --- a/main.py +++ b/main.py @@ -9,7 +9,7 @@ from utils.console import print_markdown, print_step from utils.checker import check_env # from utils.checker import envUpdate -from video_creation.background import download_background, chop_background_video +from video_creation.background import download_background, chop_background_video, get_background_config from video_creation.final_video import make_final_video from video_creation.screenshot_downloader import download_screenshots_of_reddit_posts from video_creation.voices import save_text_to_mp3 @@ -37,9 +37,10 @@ def main(POST_ID=None): length, number_of_comments = save_text_to_mp3(reddit_object) length = math.ceil(length) download_screenshots_of_reddit_posts(reddit_object, number_of_comments) - download_background() - chop_background_video(length) - make_final_video(number_of_comments, length, reddit_object) + bg_config = get_background_config() + download_background(bg_config) + chop_background_video(bg_config, length) + make_final_video(number_of_comments, length, reddit_object, bg_config) def run_many(times): diff --git a/video_creation/background.py b/video_creation/background.py index 2654499..62d1360 100644 --- a/video_creation/background.py +++ b/video_creation/background.py @@ -1,15 +1,53 @@ import random -from os import listdir, environ +from os import listdir, environ, getenv from pathlib import Path +import random from random import randrange -from typing import Tuple +from typing import Any, Tuple + +from dotenv import load_dotenv from moviepy.editor import VideoFileClip from moviepy.video.io.ffmpeg_tools import ffmpeg_extract_subclip from pytube import YouTube +from pytube.cli import on_progress from utils.console import print_step, print_substep +# Supported Background. Can add/remove background video here.... +# - : key -> used as keyword for .env file. value -> background configuration +# Format (value): +# 1. Youtube URI +# 2. filename +# 3. Citation (owner of the video) +# 4. Position of image clips in the background. See moviepy reference for more information. (https://zulko.github.io/moviepy/ref/VideoClip/VideoClip.html#moviepy.video.VideoClip.VideoClip.set_position) +background_options = { + "motor-gta": ( # Motor-GTA Racing + "https://www.youtube.com/watch?v=vw5L4xCPy9Q", + "bike-parkour-gta.mp4", + "Achy Gaming", + lambda t: ('center', 480 + t) + ), + "rocket-league": ( # Rocket League + "https://www.youtube.com/watch?v=2X9QGY__0II", + "rocket_league.mp4", + "Orbital Gameplay", + "top" + ), + "minecraft": ( # Minecraft parkour + "https://www.youtube.com/watch?v=n_Dv4JMiwK8", + "parkour.mp4", + "bbswitzer", + "center" + ), + "gta": ( # GTA Stunt Race + "https://www.youtube.com/watch?v=qGa9kWREOnE", + "gta-stunt-race.mp4", + "Achy Gaming", + lambda t: ('center', 480 + t) + ) +} + def get_start_and_end_times(video_length: int, length_of_clip: int) -> Tuple[int, int]: """Generates a random interval of time to be used as the background of the video. @@ -25,49 +63,58 @@ def get_start_and_end_times(video_length: int, length_of_clip: int) -> Tuple[int return random_time, random_time + video_length -def download_background(): - """Downloads the backgrounds/s video from YouTube.""" +def get_background_config(): + """Fetch the background/s configuration""" + load_dotenv() + try: + choice = getenv("BackgroundChoice").casefold() + except AttributeError: + print_substep("No background selected. Picking random background'") + choice = None + + # Handle default / not supported background using default option. + # Default : pick random from supported background. + if not choice or choice not in background_options: + choice = random.choice(list(background_options.keys())) + + return background_options[choice] + + +def download_background(background_config: Tuple[str, str, str, Any]): + """Downloads the background/s video from YouTube.""" Path("./assets/backgrounds/").mkdir(parents=True, exist_ok=True) - background_options = [ # uri , filename , credit - ("https://www.youtube.com/watch?v=n_Dv4JMiwK8", "parkour.mp4", "bbswitzer"), - # ( - # "https://www.youtube.com/watch?v=2X9QGY__0II", - # "rocket_league.mp4", - # "Orbital Gameplay", - # ), - ] # note: make sure the file name doesn't include an - in it - if not len(listdir("./assets/backgrounds")) >= len( - background_options - ): # if there are any background videos not installed - print_step( - "We need to download the backgrounds videos. they are fairly large but it's only done once. 😎" - ) - print_substep("Downloading the backgrounds videos... please be patient 🙏 ") - for uri, filename, credit in background_options: - if Path(f"assets/backgrounds/{credit}-{filename}").is_file(): - continue # adds check to see if file exists before downloading - print_substep(f"Downloading {filename} from {uri}") - YouTube(uri).streams.filter(res="1080p").first().download( - "assets/backgrounds", filename=f"{credit}-{filename}" - ) + uri, filename, credit, _ = background_config + if Path(f"assets/backgrounds/{credit}-{filename}").is_file(): + return + print_step( + "We need to download the backgrounds videos. they are fairly large but it's only done once. 😎" + ) + print_substep("Downloading the backgrounds videos... please be patient 🙏 ") + print_substep(f"Downloading {filename} from {uri}") + YouTube(uri, on_progress_callback=on_progress).streams.filter(res="1080p").first().download( + "assets/backgrounds", filename=f"{credit}-{filename}" + ) + print_substep("Background videos downloaded successfully! 🎉", + style="bold green") - print_substep("Background videos downloaded successfully! 🎉", style="bold green") - -def chop_background_video(video_length: int): +def chop_background_video(background_config: Tuple[str, str, str, Any], video_length: int): """Generates the background footage to be used in the video and writes it to assets/temp/background.mp4 Args: + background_config (Tuple[str, str, str, Any]) : Current background configuration video_length (int): Length of the clip where the background footage is to be taken out of """ + print_step("Finding a spot in the backgrounds video to chop...✂️") - choice = random.choice(listdir("assets/backgrounds")) + choice = f"{background_config[2]}-{background_config[1]}" environ["background_credit"] = choice.split("-")[0] background = VideoFileClip(f"assets/backgrounds/{choice}") - start_time, end_time = get_start_and_end_times(video_length, background.duration) + start_time, end_time = get_start_and_end_times( + video_length, background.duration) try: ffmpeg_extract_subclip( f"assets/backgrounds/{choice}", diff --git a/video_creation/final_video.py b/video_creation/final_video.py index 84698c5..f2f12c8 100755 --- a/video_creation/final_video.py +++ b/video_creation/final_video.py @@ -4,6 +4,7 @@ import os import re from os.path import exists from typing import Dict +from typing import Tuple, Any from moviepy.editor import ( VideoFileClip, @@ -21,12 +22,13 @@ from utils.cleanup import cleanup from utils.console import print_step, print_substep from utils.videos import save_data + console = Console() W, H = 1080, 1920 -def make_final_video(number_of_clips: int, length: int, reddit_obj: dict): +def make_final_video(number_of_clips: int, length: int, reddit_obj: dict[str], background_config: Tuple[str, str, str, Any]): """Gathers audio clips, gathers all screenshots, stitches them together and saves the final video to assets/temp Args: @@ -55,26 +57,37 @@ def make_final_video(number_of_clips: int, length: int, reddit_obj: dict): # add title to video image_clips = [] # Gather all images - new_opacity = 1 if opacity is None or float(opacity) >= 1 else float(opacity) - - image_clips.insert( - 0, - ImageClip("assets/temp/png/title.png") - .set_duration(audio_clips[0].duration) - .set_position("center") - .resize(width=W - 100) - .set_opacity(new_opacity) - ) - - for i in range(0, number_of_clips): - image_clips.append( - ImageClip(f"assets/temp/png/comment_{i}.png") - .set_duration(audio_clips[i + 1].duration) - .set_position("center") + if opacity is None or float(opacity) >= 1: # opacity not set or is set to one OR MORE + image_clips.insert( + 0, + ImageClip("assets/temp/png/title.png") + .set_duration(audio_clips[0].duration) + .resize(width=W - 100), + ) + else: + image_clips.insert( + 0, + ImageClip("assets/temp/png/title.png") + .set_duration(audio_clips[0].duration) .resize(width=W - 100) .set_opacity(new_opacity) ) + for i in range(0, number_of_clips): + if opacity is None or float(opacity) >= 1: # opacity not set or is set to one OR MORE + image_clips.append( + ImageClip(f"assets/temp/png/comment_{i}.png") + .set_duration(audio_clips[i + 1].duration) + .resize(width=W - 100), + ) + else: + image_clips.append( + ImageClip(f"assets/temp/png/comment_{i}.png") + .set_duration(audio_clips[i + 1].duration) + .resize(width=W - 100) + .set_opacity(float(opacity)), + ) + # if os.path.exists("assets/mp3/posttext.mp3"): # image_clips.insert( # 0, @@ -85,7 +98,9 @@ def make_final_video(number_of_clips: int, length: int, reddit_obj: dict): # .set_opacity(float(opacity)), # ) # else: - image_concat = concatenate_videoclips(image_clips).set_position(("center", "center")) + img_clip_pos = background_config[3] + image_concat = concatenate_videoclips( + image_clips).set_position(img_clip_pos) image_concat.audio = audio_composite final = CompositeVideoClip([background_clip, image_concat]) title = re.sub(r"[^\w\s-]", "", reddit_obj["thread_title"]) From 248010df9e7815b5bfae3cfc1be1dadeebf01a73 Mon Sep 17 00:00:00 2001 From: CordlessCoder <42666308+CordlessCoder@users.noreply.github.com> Date: Sun, 3 Jul 2022 17:39:30 +0300 Subject: [PATCH 021/126] Update .env.template --- .env.template | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.env.template b/.env.template index ce28209..1cb62cb 100644 --- a/.env.template +++ b/.env.template @@ -83,7 +83,7 @@ TIKTOK_VOICE="en_us_006" #OPTIONAL BackgroundChoice="minecraft" -#EXPLANATION Sets the background for the video. Current available option : (minecraft,gta,rocket-league,motor-gta), but you can add other video easily (1080p). Check the file for more information on different background. +# EXPLANATION Sets the background for the video. Current available option : (minecraft,gta,rocket-league,motor-gta), but you can add other video easily (1080p). Check the file for more information on different background. #OPTIONAL STORYMODE="False" From 59d07f51c8719dc088313b55f263dc1bea19ecce Mon Sep 17 00:00:00 2001 From: William9923 Date: Mon, 4 Jul 2022 04:34:08 +0700 Subject: [PATCH 022/126] chore: fixup changes after rebase --- video_creation/final_video.py | 43 ++++++++++++----------------------- 1 file changed, 14 insertions(+), 29 deletions(-) diff --git a/video_creation/final_video.py b/video_creation/final_video.py index f2f12c8..428dd67 100755 --- a/video_creation/final_video.py +++ b/video_creation/final_video.py @@ -3,7 +3,6 @@ import multiprocessing import os import re from os.path import exists -from typing import Dict from typing import Tuple, Any from moviepy.editor import ( @@ -28,7 +27,7 @@ console = Console() W, H = 1080, 1920 -def make_final_video(number_of_clips: int, length: int, reddit_obj: dict[str], background_config: Tuple[str, str, str, Any]): +def make_final_video(number_of_clips: int, length: int, reddit_obj: dict, background_config: Tuple[str, str, str, Any]): """Gathers audio clips, gathers all screenshots, stitches them together and saves the final video to assets/temp Args: @@ -57,37 +56,23 @@ def make_final_video(number_of_clips: int, length: int, reddit_obj: dict[str], b # add title to video image_clips = [] # Gather all images - if opacity is None or float(opacity) >= 1: # opacity not set or is set to one OR MORE - image_clips.insert( - 0, - ImageClip("assets/temp/png/title.png") - .set_duration(audio_clips[0].duration) - .resize(width=W - 100), - ) - else: - image_clips.insert( - 0, - ImageClip("assets/temp/png/title.png") - .set_duration(audio_clips[0].duration) + new_opacity = 1 if opacity is None or float(opacity) >= 1 else float(opacity) + image_clips.insert( + 0, + ImageClip("assets/temp/png/title.png") + .set_duration(audio_clips[0].duration) + .resize(width=W - 100) + .set_opacity(new_opacity) + ) + + for i in range(0, number_of_clips): + image_clips.append( + ImageClip(f"assets/temp/png/comment_{i}.png") + .set_duration(audio_clips[i + 1].duration) .resize(width=W - 100) .set_opacity(new_opacity) ) - for i in range(0, number_of_clips): - if opacity is None or float(opacity) >= 1: # opacity not set or is set to one OR MORE - image_clips.append( - ImageClip(f"assets/temp/png/comment_{i}.png") - .set_duration(audio_clips[i + 1].duration) - .resize(width=W - 100), - ) - else: - image_clips.append( - ImageClip(f"assets/temp/png/comment_{i}.png") - .set_duration(audio_clips[i + 1].duration) - .resize(width=W - 100) - .set_opacity(float(opacity)), - ) - # if os.path.exists("assets/mp3/posttext.mp3"): # image_clips.insert( # 0, From fd80b8833dded94aa74cedf5f09cb281d3a9bcb3 Mon Sep 17 00:00:00 2001 From: micziz Date: Tue, 5 Jul 2022 12:29:02 +0200 Subject: [PATCH 023/126] THis shloud work! --- install.sh | 220 +++++++++++++++++++++++++++++++++++++++++++++++ requirements.txt | 1 + 2 files changed, 221 insertions(+) create mode 100644 install.sh diff --git a/install.sh b/install.sh new file mode 100644 index 0000000..6453dc0 --- /dev/null +++ b/install.sh @@ -0,0 +1,220 @@ +#!/bin/bash + +# If the install fails, then print an error and exit. +function install_fail() { + echo "Installation failed" + exit 1 +} + +# This is the help fuction. It helps users withe the options +function Help(){ + echo "Usage: install.sh [option]" + echo "Options:" + echo " -h: Show this help message and exit" + echo " -d: Install only dependencies" + echo " -p: Install only python dependencies (including playwright)" + echo " -b: Install just the bot" + echo " -l: Install the bot and the python dependencies" +} + +# Options +while getopts ":hydpbl" option; do + case $option in + # -h, prints help message + h) + Help exit 0;; + # -y, assumes yes + y) + ASSUME_YES=1;; + # -d install only dependencies + d) + DEPS_ONLY=1;; + # -p install only python dependencies + p) + PYTHON_ONLY=1;; + b) + JUST_BOT=1;; + l) + BOT_AND_PYTHON=1;; + # if a bad argument is given, then throw an error + \?) + echo "Invalid option: -$OPTARG" >&2 Help exit 1;; + :) + echo "Option -$OPTARG requires an argument." >&2 Help exit 1;; + esac +done + +# Install dependencies for MacOS +function install_macos(){ + # Check if homebrew is installed + if [ ! command -v brew &> /dev/null ]; then + echo "Installing Homebrew" + # if it's is not installed, then install it in a NONINTERACTIVE way + NONINTERACTIVE=1 /bin/bash -c "$(curl -fsSL https://raw.githubusercontent.com/Homebrew/install/HEAD/uninstall.sh)" + # Check for what arcitecture, so you can place path. + if [[ "uname -m" == "x86_64" ]]; then + echo "export PATH=/usr/local/bin:$PATH" >> ~/.bash_profile && source ~/.bash_profile + fi + # If not + else + # Print that it's already installed + echo "Homebrew is already installed" + fi + # Install the required packages + echo "Installing required Packages" + brew install python@3.10 tcl-tk python-tk +} + +# Function to install for arch (and other forks like manjaro) +function install_arch(){ + echo "Installing required packages" + sudo pacman -S --needed python3 tk git && python3 -m ensurepip || install_fail +} + +# Function to install for debian (and ubuntu) +function install_deb(){ + echo "Installing required packages" + sudo apt install python3 python3-dev python3-tk python3-pip git || install_fail +} + +# Function to install for fedora (and other forks) +function install_fedora(){ + echo "Installing required packages" + sudo dnf install python3 python3-tkinter python3-pip git python3-devel || install_fail +} + +# Function to install for centos (and other forks based on it) +function install_centos(){ + echo "Installing required packages" + sudo yum install -y python3 || install_fail + sudo yum install -y python3-tkinter epel-release python3-pip git || install_fail +} + +function get_the_bot(){ + echo "Downloading the bot" + git clone https://github.com/elebumm/RedditVideoMakerBot.git +} + +#install python dependencies +function install_python_dep(){ + # tell the user that the script is going to install the python dependencies + echo "Installing python dependencies" + # cd into the directory + cd RedditVideoMakerBot + # install the dependencies + pip3 install -r requirements.txt + # cd out + cd .. +} + +# install playwright function +function install_playwright(){ + # tell the user that the script is going to install playwright + echo "Installing playwright" + # cd into the directory where the script is downloaded + cd RedditVideoMakerBot + # run the install script + playwright install + playwright install-deps + # give a note + printf "Note, if these gave any errors, please run this command This will (maybe) fix the issue with playwright.\npython -m playwright install && python -m playwright install-deps" + cd .. +} + +# Install depndencies +function install_deps(){ + # if the platform is mac, install macos + if [ "$(uname)" == "Darwin" ]; then + install_macos || install_fail + # if pacman is found + elif [ -x "$(command -v pacman)" ]; then + # install for arch + install_arch || install_fail + # if apt-get is found + elif [ -x "$(command -v apt-get)" ]; then + # install fro debian + install_deb || install_fail + # if dnf is found + elif [ -x "$(command -v dnf)" ]; then + # install for fedora + install_fedora || install_fail + # if yum is found + elif [ -x "$(command -v yum)" ]; then + # install for centos + install_centos || install_fail + # else + else + # print an error message and exit + printf "Your OS is not supported\n Please install python3, pip3 and git manually\n After that, run the script again with the -pb option to install python and playwright dependencies\n If you want to add support for your OS, please open a pull request on github\n +https://github.com/elebumm/RedditVideoMakerBot" + exit 1 + fi +} + +# Main function +function install_main(){ + # Print that are installing + echo "Installing..." + # if -y (assume yes) continue + if [[ ASSUME_YES -eq 1 ]]; then + echo "Assuming yes" + # else, ask if they want to continue + else + echo "Continue? (y/n)" + read answer + # if the answer is not yes, then exit + if [ "$answer" != "y" ]; then + echo "Aborting" + exit 1 + fi + fi + # if the -d (only dependencies) options is selected install just the dependencies + if [[ DEPS_ONLY -eq 1 ]]; then + echo "Installing only dependencies" + install_deps + elif [[ PYTHON_ONLY -eq 1 ]]; then + # if the -p (only python dependencies) options is selected install just the python dependencies and playwright + echo "Installing only python dependencies" + install_python_dep + install_playwright + # if the -b (only the bot) options is selected install just the bot + elif [[ JUST_BOT -eq 1 ]]; then + echo "Installing only the bot" + get_the_bot + # if the -l (bot and python) options is selected install just the bot and python dependencies + elif [[ BOT_AND_PYTHON -eq 1 ]]; then + echo "Installing only the bot and python dependencies" + get_the_bot + install_python_dep + # else, install everything + else + echo "Installing all" + install_deps + get_the_bot + install_python_dep + install_playwright + fi + + DIR="./RedditVideoMakerBot" + if [ -d "$DIR" ]; then + printf "\nThe bot is already installed, want to run it?" + # if -y (assume yes) continue + if [[ ASSUME_YES -eq 1 ]]; then + echo "Assuming yes" + # else, ask if they want to continue + else + echo "Continue? (y/n)" + read answer + # if the answer is not yes, then exit + if [ "$answer" != "y" ]; then + echo "Aborting" + exit 1 + fi + fi + cd RedditVideoMakerBot + python3 main.py + fi +} + +# Run the main function +install_main \ No newline at end of file diff --git a/requirements.txt b/requirements.txt index 687f952..22b53b7 100644 --- a/requirements.txt +++ b/requirements.txt @@ -10,3 +10,4 @@ pytube==12.1.0 requests==2.28.1 rich==12.4.4 translators==5.3.1 +sox==1.4.1 From 74387ef8738dc877fe3b586276151027793b0004 Mon Sep 17 00:00:00 2001 From: micziz Date: Tue, 5 Jul 2022 12:36:45 +0200 Subject: [PATCH 024/126] Modify readme --- README.md | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/README.md b/README.md index dc6237c..dffef91 100644 --- a/README.md +++ b/README.md @@ -42,6 +42,12 @@ The only original thing being done is the editing and gathering of all materials 3. Run `playwright install` and `playwright install-deps`. (if this fails try adding python -m to the front of the command) +**EXPERIMENTAL!!!!** + +On MacOS and Linux (debian, arch, fedora and centos, and based on those), you can run an install script that will automatically install steps 1 to 3. (requires bash) + +`bash <(curl -sL https://raw.githubusercontent.com/micziz/RedditVideoMakerBot/master/install.sh)` + 4. Run `python main.py` required\*\*), visit [the Reddit Apps page.](https://www.reddit.com/prefs/apps) TL;DR set up an app that is a "script". 5. Enjoy 😎 From c3544bf64e621289a40f09604feba96e4eff3775 Mon Sep 17 00:00:00 2001 From: micziz Date: Tue, 5 Jul 2022 12:44:09 +0200 Subject: [PATCH 025/126] Final fix in readme --- README.md | 2 ++ 1 file changed, 2 insertions(+) diff --git a/README.md b/README.md index dffef91..dad8f59 100644 --- a/README.md +++ b/README.md @@ -48,6 +48,8 @@ On MacOS and Linux (debian, arch, fedora and centos, and based on those), you ca `bash <(curl -sL https://raw.githubusercontent.com/micziz/RedditVideoMakerBot/master/install.sh)` +This can also be used to update the installation + 4. Run `python main.py` required\*\*), visit [the Reddit Apps page.](https://www.reddit.com/prefs/apps) TL;DR set up an app that is a "script". 5. Enjoy 😎 From 1784ea47be07ea76cdead58ac75b6f560e971685 Mon Sep 17 00:00:00 2001 From: Callum Leslie Date: Tue, 5 Jul 2022 15:09:52 +0100 Subject: [PATCH 026/126] docs: update contributing guidelines to use rebase --- CONTRIBUTING.md | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md index e3e858d..b51f91f 100644 --- a/CONTRIBUTING.md +++ b/CONTRIBUTING.md @@ -111,7 +111,8 @@ When making your PR, follow these guidelines: - Your branch has a base of _develop_, **not** _master_ - You are merging your branch into the _develop_ branch -- You link any issues that are resolved or fixed by your changes. (this is done by typing "Fixes #\") in your pull request. +- You link any issues that are resolved or fixed by your changes. (this is done by typing "Fixes #\") in your pull request +- Where possible, you have used `git pull --rebase`, to avoid creating unnecessary merge commits ### Improving The Documentation From 3de18b67190a8a381415f01da8d24131b49eed5a Mon Sep 17 00:00:00 2001 From: Callum Leslie Date: Tue, 5 Jul 2022 15:22:11 +0100 Subject: [PATCH 027/126] docs: commit style --- CONTRIBUTING.md | 8 ++++++++ 1 file changed, 8 insertions(+) diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md index b51f91f..ca1c7cb 100644 --- a/CONTRIBUTING.md +++ b/CONTRIBUTING.md @@ -113,6 +113,14 @@ When making your PR, follow these guidelines: - You are merging your branch into the _develop_ branch - You link any issues that are resolved or fixed by your changes. (this is done by typing "Fixes #\") in your pull request - Where possible, you have used `git pull --rebase`, to avoid creating unnecessary merge commits +- You have meaningful commits, and if possible, follow the commit style guide of `type: explanation` +- Here are the commit types: + - **feat** - a new feature + - **fix** - a bug fix + - **docs** - a change to documentation / commenting + - **style** - formatting changes - does not impact code + - **refactor** - refactored code + - **chore** - updating configs, workflows etc - does not impact code ### Improving The Documentation From eefd56e16f022b86c7bcb0eb1f5b4e5dfcd249b7 Mon Sep 17 00:00:00 2001 From: CordlessCoder <42666308+CordlessCoder@users.noreply.github.com> Date: Tue, 5 Jul 2022 16:18:17 +0000 Subject: [PATCH 028/126] Add files via upload --- utils/console.py | 85 +++++++++----- utils/tomlchecker.py | 262 +++++++++++++++++++++++++++++++++++++++++++ 2 files changed, 318 insertions(+), 29 deletions(-) create mode 100644 utils/tomlchecker.py diff --git a/utils/console.py b/utils/console.py index 5b91fef..8a68280 100644 --- a/utils/console.py +++ b/utils/console.py @@ -4,7 +4,6 @@ from rich.markdown import Markdown from rich.padding import Padding from rich.panel import Panel from rich.text import Text -from rich.columns import Columns import re console = Console() @@ -29,12 +28,6 @@ def print_substep(text, style=""): console.print(text, style=style) -def print_table(items): - """Prints items in a table.""" - - console.print(Columns([Panel(f"[yellow]{item}", expand=True) for item in items])) - - def handle_input( message: str = "", check_type=False, @@ -44,33 +37,67 @@ def handle_input( nmax=None, oob_error="", extra_info="", + options: list = None, + default=NotImplemented, ): - match = re.compile(match + "$") - console.print(extra_info, no_wrap=True) - while True: - console.print(message, end="") - user_input = input("").strip() - if re.match(match, user_input) is not None: + if default is not NotImplemented: + console.print( + "[green]" + + message + + '\n[blue bold]The default value is "' + + str(default) + + '"\nDo you want to use it?(y/n)' + ) + if input().casefold().startswith("y"): + return default + if options is None: + match = re.compile(match) + console.print("[green bold]" + extra_info, no_wrap=True) + while True: + console.print(message, end="") + user_input = input("").strip() if check_type is not False: try: - user_input = check_type(user_input) # this line is fine - if nmin is not None and user_input < nmin: - console.print("[red]" + oob_error) # Input too low failstate - continue - if nmax is not None and user_input > nmax: - console.print("[red]" + oob_error) # Input too high + user_input = check_type(user_input) + if (nmin is not None and user_input < nmin) or ( + nmax is not None and user_input > nmax + ): + # FAILSTATE Input out of bounds + console.print("[red]" + oob_error) continue break # Successful type conversion and number in bounds except ValueError: - console.print("[red]" + err_message) # Type conversion failed + # Type conversion failed + console.print("[red]" + err_message) continue - if nmin is not None and len(user_input) < nmin: # Check if string is long enough - console.print("[red]" + oob_error) + elif match != "" and re.match(match, user_input) is None: + console.print( + "[red]" + err_message + + "\nAre you absolutely sure it's correct?(y/n)" + ) + if input().casefold().startswith("y"): + break continue - if nmax is not None and len(user_input) > nmax: # Check if string is not too long - console.print("[red]" + oob_error) - continue - break - console.print("[red]" + err_message) - - return user_input + else: + # FAILSTATE Input STRING out of bounds + if (nmin is not None and len(user_input) < nmin) or ( + nmax is not None and len(user_input) > nmax + ): + console.print("[red bold]" + oob_error) + continue + break # SUCCESS Input STRING in bounds + return user_input + console.print(extra_info, no_wrap=True) + while True: + console.print(message, end="") + user_input = input("").strip() + if user_input not in options: + console.print( + "[red bold]" + + err_message + + "\nValid options are: " + + ", ".join(map(str, options)) + + "." + ) + continue + return user_input diff --git a/utils/tomlchecker.py b/utils/tomlchecker.py new file mode 100644 index 0000000..c44108f --- /dev/null +++ b/utils/tomlchecker.py @@ -0,0 +1,262 @@ +#!/usr/bin/env python +# import os +import toml +from rich import pretty +from rich.console import Console +import re + +# from utils.console import handle_input +from console import handle_input + + +console = Console() + + +printed = False + + +def crawl(obj: dict, func=lambda x, y: print(x, y, end="\n"), path: list = []): + for key in obj.keys(): + if type(obj[key]) is dict: + crawl(obj[key], func, path + [key]) + continue + func(path + [key], obj[key]) + + +def check(value, checks, name): + global printed + if printed is False: + console.print( + """\ +[blue bold]############################### +# # +# Checking TOML configuration # +# # +############################### +If you see any prompts, that means that you have unset/incorrectly set variables, please input the correct values.\ +""" + ) + printed = True + if "type" in checks: + try: + value = eval(checks["type"])(value) + except: + value = handle_input( + message=( + ( + ("[blue]Example: " + str(checks["example"]) + "\n") + if "example" in checks + else "" + ) + + "[red]" + + ("Non-optional ", "Optional ")[ + "optional" in checks and checks["optional"] is True + ] + ) + + " [#C0CAF5 bold]" + + str(name) + + "[#F7768E bold]=", + check_type=eval(checks["type"]), + extra_info=checks["explanation"] if "explanation" in checks else "", + default=checks["default"] if "default" in checks else NotImplemented, + match=checks["regex"] if "regex" in checks else "", + err_message=checks["input_error"] if "input_error" in checks else "Incorrect input", + nmin=checks["nmin"] if "nmin" in checks else None, + nmax=checks["nmax"] if "nmax" in checks else None, + oob_error=checks["oob_error"] + if "oob_error" in checks + else "Input out of bounds(Value too high/low/long/short)", + ) + + if ( + "options" in checks and value not in checks["options"] + ): # FAILSTATE Value is not one of the options + value = handle_input( + message=( + (("[blue]Example: " + str(checks["example"]) + "\n") + if "example" in checks else "") + + "[red]" + + ("Non-optional ", "Optional ")[ + "optional" in checks and checks["optional"] is True + ] + ) + + "[#C0CAF5 bold]" + + str(name) + + "[#F7768E bold]=", + extra_info=checks["explanation"] if "explanation" in checks else "", + err_message=checks["input_error"] if "input_error" in checks else "Incorrect input", + default=checks["default"] if "default" in checks else NotImplemented, + options=checks["options"], + ) + if "regex" in checks and ( + (isinstance(value, str) and re.match(checks["regex"], value) is None) + or not isinstance(value, str) + ): # FAILSTATE Value doesn't match regex, or has regex but is not a string. + value = handle_input( + message=( + (("[blue]Example: " + str(checks["example"]) + "\n") + if "example" in checks else "") + + "[red]" + + ("Non-optional ", "Optional ")[ + "optional" in checks and checks["optional"] is True + ] + ) + + "[#C0CAF5 bold]" + + str(name) + + "[#F7768E bold]=", + extra_info=checks["explanation"] if "explanation" in checks else "", + match=checks["regex"], + err_message=checks["input_error"] if "input_error" in checks else "Incorrect input", + default=checks["default"] if "default" in checks else NotImplemented, + nmin=checks["nmin"] if "nmin" in checks else None, + nmax=checks["nmax"] if "nmax" in checks else None, + oob_error=checks["oob_error"] + if "oob_error" in checks + else "Input out of bounds(Value too high/low/long/short)", + ) + + if not hasattr(value, "__iter__") and ( + ("nmin" in checks and checks["nmin"] + is not None and value < checks["nmin"]) + or ("nmax" in checks and checks["nmax"] is not None and value > checks["nmax"]) + ): + value = handle_input( + message=( + (("[blue]Example: " + str(checks["example"]) + "\n") + if "example" in checks else "") + + "[red]" + + ("Non-optional ", "Optional ")[ + "optional" in checks and checks["optional"] is True + ] + ) + + "[#C0CAF5 bold]" + + str(name) + + "[#F7768E bold]=", + extra_info=checks["explanation"] if "explanation" in checks else "", + default=checks["default"] if "default" in checks else NotImplemented, + match=checks["regex"] if "regex" in checks else "", + err_message=checks["input_error"] if "input_error" in checks else "Incorrect input", + nmin=checks["nmin"] if "nmin" in checks else None, + nmax=checks["nmax"] if "nmax" in checks else None, + oob_error=checks["oob_error"] + if "oob_error" in checks + else "Input out of bounds(Value too high/low/long/short)", + ) + if hasattr(value, "__iter__") and ( + ("nmin" in checks and checks["nmin"] + is not None and len(value) < checks["nmin"]) + or ("nmax" in checks and checks["nmax"] is not None and len(value) > checks["nmax"]) + ): + value = handle_input( + message=( + (("[blue]Example: " + str(checks["example"]) + "\n") + if "example" in checks else "") + + "[red]" + + ("Non-optional ", "Optional ")[ + "optional" in checks and checks["optional"] is True + ] + ) + + "[#C0CAF5 bold]" + + str(name) + + "[#F7768E bold]=", + extra_info=checks["explanation"] if "explanation" in checks else "", + default=checks["default"] if "default" in checks else NotImplemented, + match=checks["regex"] if "regex" in checks else "", + err_message=checks["input_error"] if "input_error" in checks else "Incorrect input", + nmin=checks["nmin"] if "nmin" in checks else None, + nmax=checks["nmax"] if "nmax" in checks else None, + oob_error=checks["oob_error"] + if "oob_error" in checks + else "Input out of bounds(Value too high/low/long/short)", + ) + if value == {}: + handle_input( + message=( + (("[blue]Example: " + str(checks["example"]) + "\n") + if "example" in checks else "") + + "[red]" + + ("Non-optional ", "Optional ")[ + "optional" in checks and checks["optional"] is True + ] + ) + + "[#C0CAF5 bold]" + + str(name) + + "[#F7768E bold]=", + extra_info=checks["explanation"] if "explanation" in checks else "", + default=checks["default"] if "default" in checks else NotImplemented, + match=checks["regex"] if "regex" in checks else "", + err_message=checks["input_error"] if "input_error" in checks else "Incorrect input", + nmin=checks["nmin"] if "nmin" in checks else None, + nmax=checks["nmax"] if "nmax" in checks else None, + oob_error=checks["oob_error"] + if "oob_error" in checks + else "Input out of bounds(Value too high/low/long/short)", + ) + return value + + +def crawl_and_check(obj: dict, path: list, checks: dict = {}, name=""): + if len(path) == 0: + return check(obj, checks, name) + if path[0] not in obj.keys(): + obj[path[0]] = {} + obj[path[0]] = crawl_and_check(obj[path[0]], path[1:], checks, path[0]) + return obj + + +def check_vars(path, checks): + global config + crawl_and_check(config, path, checks) + + +def check_toml(template_file, config_file) -> bool: + try: + template = toml.load(template_file) + except Exception as error: + console.print( + f"[red bold]Encountered error when trying to to load {template_file}: {error}" + ) + return False + try: + global config + config = toml.load(config_file) + except (toml.TomlDecodeError): + console.print( + f"""[blue]Couldn't read {config_file}. +Overwrite it?(y/n)""" + ) + if not input().startswith("y"): + print("Unable to read config, and not allowed to overwrite it. Giving up.") + return False + else: + try: + with open(config_file, "w") as f: + f.write("") + except: + console.print( + f"[red bold]Failed to overwrite {config_file}. Giving up.\nSuggestion: check {config_file} permissions for the user." + ) + return False + except (FileNotFoundError): + console.print( + f"""[blue]Couldn't find {config_file} +Creating it now.""" + ) + try: + with open(config_file, "x") as f: + f.write("") + config = {} + except: + console.print( + f"[red bold]Failed to write to {config_file}. Giving up.\nSuggestion: check the folder's permissions for the user." + ) + return False + crawl(template, check_vars) + pretty.pprint(config) + with open(config_file, "w") as f: + toml.dump(config, f) + return True + + +if __name__ == "__main__": + check_toml(".config.template.toml", "config.toml") From e91209fd533192cedf009f64b9cc16183b819c82 Mon Sep 17 00:00:00 2001 From: CordlessCoder <42666308+CordlessCoder@users.noreply.github.com> Date: Tue, 5 Jul 2022 19:20:06 +0300 Subject: [PATCH 029/126] Update console.py --- utils/console.py | 8 ++++++++ 1 file changed, 8 insertions(+) diff --git a/utils/console.py b/utils/console.py index 8a68280..fdab5d5 100644 --- a/utils/console.py +++ b/utils/console.py @@ -4,6 +4,7 @@ from rich.markdown import Markdown from rich.padding import Padding from rich.panel import Panel from rich.text import Text +from rich.columns import Columns import re console = Console() @@ -22,7 +23,14 @@ def print_step(text): panel = Panel(Text(text, justify="left")) console.print(panel) + +def print_table(items): + """Prints items in a table.""" + console.print(Columns([Panel(f"[yellow]{item}", expand=True) for item in items])) + + + def print_substep(text, style=""): """Prints a rich info message without the panelling.""" console.print(text, style=style) From c8817d9ffd1145678d35101badbe9f46a876d9cd Mon Sep 17 00:00:00 2001 From: CordlessCoder Date: Tue, 5 Jul 2022 20:02:04 +0300 Subject: [PATCH 030/126] painstakingly restored .config.template.toml --- .config.template.toml | 41 +++++++++++++++++++++++++++++++++++++++++ 1 file changed, 41 insertions(+) create mode 100644 .config.template.toml diff --git a/.config.template.toml b/.config.template.toml new file mode 100644 index 0000000..e7de56f --- /dev/null +++ b/.config.template.toml @@ -0,0 +1,41 @@ +[reddit.creds] +client_id = { optional = false, nmin = 12, nmax = 30, explanation = "the ID of your Reddit app of SCRIPT type", example = "fFAGRNJru1FTz70BzhT3Zg", regex = "^[-a-zA-Z0-9._~+/]+=*$", input_error = "The client ID can only contain printable characters.", oob_error = "The ID should be over 12 and under 30 characters, double check your input." } +client_secret = { optional = false, nmin = 20, nmax = 40, explanation = "the SECRET of your Reddit app of SCRIPT type", example = "fFAGRNJru1FTz70BzhT3Zg", regex = "^[-a-zA-Z0-9._~+/]+=*$", input_error = "The client ID can only contain printable characters.", oob_error = "The secret should be over 20 and under 40 characters, double check your input." } +username = { optional = false, nmin = 3, nmax = 20, explanation = "the username of your reddit account", example = "asdfghjkl", regex = "^[-_0-9a-zA-Z]+$", oob_error = "A username HAS to be between 3 and 20 characters" } +password = { optional = false, nmin = 8, explanation = "the password of your reddit account", example = "fFAGRNJru1FTz70BzhT3Zg", oob_error = "Password too short" } +2fa = { optional = true, options = [ + true, + false, +], default = false, explanation = "Whether you have Reddit 2FA enabled, Valid options are True and Talse", example = true } + + +[reddit.thread] +random = { optional = true, options = [ + true, + false, +], default = false, explanation = "If set to no, it will ask you a thread link to extract the thread, if yes it will randomize it. Default: 'False'", example = "True" } +subreddit = { optional = false, regex = "[_0-9a-zA-Z]+$", nmin = 3, nmax = 21, explanation = "what subreddit to pull posts from, the name of the sub, not the URL", example = "AskReddit", oob_error = "A subreddit name HAS to be between 3 and 20 characters" } +post_id = { optional = false, regex = "^((?!://|://).)*$", explanation = "Used if you want to use a specific post.", example = "urdtfx" } +max_comment_length = { default = 500, optional = false, nmin = 10, nmax = 10000, type = "int", explanation = "max number of characters a comment can have. default is 500", example = 500, oob_error = "the max comment length should be between 10 and 10000" } + + +[settings] +allow_nsfw = { optional = true, default = false, example = false, options = [ + true, + false, +], explanation = "Whether to allow NSFW content, True or False" } +theme = { optional = true, default = "light", example = "dark", options = [ + "dark", + "light", +], explanation = "sets the Reddit theme, either LIGHT or DARK" } +times_to_run = { optional = true, default = 1, example = 2, explanation = "used if you want to run multiple times. set to an int e.g. 4 or 29 or 1", type = "int", nmin = 1, oob_error = "It's very hard to run something less than once." } +opacity = { optional = true, default = 0.9, example = 0.8, explanation = "Sets the opacity of the comments when overlayed over the background", type = "float", nmin = 0, nmax = 1, oob_error = "The opacity HAS to be between 0 and 1", input_error = "The opacity HAS to be a decimal number between 0 and 1" } +storymode = { optional = true, default = false, example = false, options = [ + true, + false, +] } + + +# [settings.tts] +# voice = { optional = true, example = "en_us_002", explanation = "sets the voice used by TTS" } +# choice = { optional = true, example = "polly", explanation = "the backend used " } From ad30e9a8d9ac35e77ffe6e9316a1d1f93340b2d1 Mon Sep 17 00:00:00 2001 From: CordlessCoder Date: Tue, 5 Jul 2022 20:03:52 +0300 Subject: [PATCH 031/126] removed a debug print statement --- config.toml | 19 +++++++++++++++++++ utils/tomlchecker.py | 23 ++++++++--------------- 2 files changed, 27 insertions(+), 15 deletions(-) create mode 100644 config.toml mode change 100644 => 100755 utils/tomlchecker.py diff --git a/config.toml b/config.toml new file mode 100644 index 0000000..baba72c --- /dev/null +++ b/config.toml @@ -0,0 +1,19 @@ +[settings] +allow_nsfw = false +theme = "light" +times_to_run = 1 +opacity = 0.9 +storymode = false + +[reddit.creds] +client_id = "adsfadsfasdf" +client_secret = "adsfasdfadsfasdfasdf" +username = "asdfasdfadsf" +password = "asdfasdfadsf" +2fa = false + +[reddit.thread] +random = false +subreddit = "asdfasdfadsfadsfadfs" +post_id = "asdfasdfasdf" +max_comment_length = 500 diff --git a/utils/tomlchecker.py b/utils/tomlchecker.py old mode 100644 new mode 100755 index c44108f..85f6a58 --- a/utils/tomlchecker.py +++ b/utils/tomlchecker.py @@ -73,8 +73,7 @@ If you see any prompts, that means that you have unset/incorrectly set variables ): # FAILSTATE Value is not one of the options value = handle_input( message=( - (("[blue]Example: " + str(checks["example"]) + "\n") - if "example" in checks else "") + (("[blue]Example: " + str(checks["example"]) + "\n") if "example" in checks else "") + "[red]" + ("Non-optional ", "Optional ")[ "optional" in checks and checks["optional"] is True @@ -94,8 +93,7 @@ If you see any prompts, that means that you have unset/incorrectly set variables ): # FAILSTATE Value doesn't match regex, or has regex but is not a string. value = handle_input( message=( - (("[blue]Example: " + str(checks["example"]) + "\n") - if "example" in checks else "") + (("[blue]Example: " + str(checks["example"]) + "\n") if "example" in checks else "") + "[red]" + ("Non-optional ", "Optional ")[ "optional" in checks and checks["optional"] is True @@ -116,14 +114,12 @@ If you see any prompts, that means that you have unset/incorrectly set variables ) if not hasattr(value, "__iter__") and ( - ("nmin" in checks and checks["nmin"] - is not None and value < checks["nmin"]) + ("nmin" in checks and checks["nmin"] is not None and value < checks["nmin"]) or ("nmax" in checks and checks["nmax"] is not None and value > checks["nmax"]) ): value = handle_input( message=( - (("[blue]Example: " + str(checks["example"]) + "\n") - if "example" in checks else "") + (("[blue]Example: " + str(checks["example"]) + "\n") if "example" in checks else "") + "[red]" + ("Non-optional ", "Optional ")[ "optional" in checks and checks["optional"] is True @@ -143,14 +139,12 @@ If you see any prompts, that means that you have unset/incorrectly set variables else "Input out of bounds(Value too high/low/long/short)", ) if hasattr(value, "__iter__") and ( - ("nmin" in checks and checks["nmin"] - is not None and len(value) < checks["nmin"]) + ("nmin" in checks and checks["nmin"] is not None and len(value) < checks["nmin"]) or ("nmax" in checks and checks["nmax"] is not None and len(value) > checks["nmax"]) ): value = handle_input( message=( - (("[blue]Example: " + str(checks["example"]) + "\n") - if "example" in checks else "") + (("[blue]Example: " + str(checks["example"]) + "\n") if "example" in checks else "") + "[red]" + ("Non-optional ", "Optional ")[ "optional" in checks and checks["optional"] is True @@ -172,8 +166,7 @@ If you see any prompts, that means that you have unset/incorrectly set variables if value == {}: handle_input( message=( - (("[blue]Example: " + str(checks["example"]) + "\n") - if "example" in checks else "") + (("[blue]Example: " + str(checks["example"]) + "\n") if "example" in checks else "") + "[red]" + ("Non-optional ", "Optional ")[ "optional" in checks and checks["optional"] is True @@ -252,7 +245,7 @@ Creating it now.""" ) return False crawl(template, check_vars) - pretty.pprint(config) + # pretty.pprint(config) with open(config_file, "w") as f: toml.dump(config, f) return True From 616da070080f51bc59d3c0f42d7b0192edeb9b2c Mon Sep 17 00:00:00 2001 From: Callum Leslie Date: Tue, 5 Jul 2022 18:52:28 +0100 Subject: [PATCH 032/126] chore: remove config.toml and add to gitignore --- .gitignore | 2 ++ config.toml | 19 ------------------- 2 files changed, 2 insertions(+), 19 deletions(-) delete mode 100644 config.toml diff --git a/.gitignore b/.gitignore index 4ee3693..793db5d 100644 --- a/.gitignore +++ b/.gitignore @@ -241,3 +241,5 @@ reddit-bot-351418-5560ebc49cac.json *.pyc video_creation/data/videos.json video_creation/data/envvars.txt + +config.toml diff --git a/config.toml b/config.toml deleted file mode 100644 index baba72c..0000000 --- a/config.toml +++ /dev/null @@ -1,19 +0,0 @@ -[settings] -allow_nsfw = false -theme = "light" -times_to_run = 1 -opacity = 0.9 -storymode = false - -[reddit.creds] -client_id = "adsfadsfasdf" -client_secret = "adsfasdfadsfasdfasdf" -username = "asdfasdfadsf" -password = "asdfasdfadsf" -2fa = false - -[reddit.thread] -random = false -subreddit = "asdfasdfadsfadsfadfs" -post_id = "asdfasdfasdf" -max_comment_length = 500 From 3c52d1b046c02e0284f50d3035d5e75a0a9f6f4d Mon Sep 17 00:00:00 2001 From: CordlessCoder Date: Tue, 5 Jul 2022 22:39:22 +0300 Subject: [PATCH 033/126] chore: removed duplicate handle_input calls --- .config.template.toml | 8 +- config.toml | 19 ----- main.py | 23 +++--- utils/console.py | 38 +++++---- utils/tomlchecker.py | 183 +++++++++++------------------------------- 5 files changed, 88 insertions(+), 183 deletions(-) delete mode 100644 config.toml diff --git a/.config.template.toml b/.config.template.toml index e7de56f..1c5ee43 100644 --- a/.config.template.toml +++ b/.config.template.toml @@ -3,7 +3,7 @@ client_id = { optional = false, nmin = 12, nmax = 30, explanation = "the ID of y client_secret = { optional = false, nmin = 20, nmax = 40, explanation = "the SECRET of your Reddit app of SCRIPT type", example = "fFAGRNJru1FTz70BzhT3Zg", regex = "^[-a-zA-Z0-9._~+/]+=*$", input_error = "The client ID can only contain printable characters.", oob_error = "The secret should be over 20 and under 40 characters, double check your input." } username = { optional = false, nmin = 3, nmax = 20, explanation = "the username of your reddit account", example = "asdfghjkl", regex = "^[-_0-9a-zA-Z]+$", oob_error = "A username HAS to be between 3 and 20 characters" } password = { optional = false, nmin = 8, explanation = "the password of your reddit account", example = "fFAGRNJru1FTz70BzhT3Zg", oob_error = "Password too short" } -2fa = { optional = true, options = [ +2fa = { optional = true, type = "bool", options = [ true, false, ], default = false, explanation = "Whether you have Reddit 2FA enabled, Valid options are True and Talse", example = true } @@ -13,14 +13,14 @@ password = { optional = false, nmin = 8, explanation = "the password of your red random = { optional = true, options = [ true, false, -], default = false, explanation = "If set to no, it will ask you a thread link to extract the thread, if yes it will randomize it. Default: 'False'", example = "True" } +], default = false, type = "bool", explanation = "If set to no, it will ask you a thread link to extract the thread, if yes it will randomize it. Default: 'False'", example = "True" } subreddit = { optional = false, regex = "[_0-9a-zA-Z]+$", nmin = 3, nmax = 21, explanation = "what subreddit to pull posts from, the name of the sub, not the URL", example = "AskReddit", oob_error = "A subreddit name HAS to be between 3 and 20 characters" } post_id = { optional = false, regex = "^((?!://|://).)*$", explanation = "Used if you want to use a specific post.", example = "urdtfx" } max_comment_length = { default = 500, optional = false, nmin = 10, nmax = 10000, type = "int", explanation = "max number of characters a comment can have. default is 500", example = 500, oob_error = "the max comment length should be between 10 and 10000" } [settings] -allow_nsfw = { optional = true, default = false, example = false, options = [ +allow_nsfw = { optional = true, type = "bool", default = false, example = false, options = [ true, false, ], explanation = "Whether to allow NSFW content, True or False" } @@ -30,7 +30,7 @@ theme = { optional = true, default = "light", example = "dark", options = [ ], explanation = "sets the Reddit theme, either LIGHT or DARK" } times_to_run = { optional = true, default = 1, example = 2, explanation = "used if you want to run multiple times. set to an int e.g. 4 or 29 or 1", type = "int", nmin = 1, oob_error = "It's very hard to run something less than once." } opacity = { optional = true, default = 0.9, example = 0.8, explanation = "Sets the opacity of the comments when overlayed over the background", type = "float", nmin = 0, nmax = 1, oob_error = "The opacity HAS to be between 0 and 1", input_error = "The opacity HAS to be a decimal number between 0 and 1" } -storymode = { optional = true, default = false, example = false, options = [ +storymode = { optional = true, type = "bool", default = false, example = false, options = [ true, false, ] } diff --git a/config.toml b/config.toml deleted file mode 100644 index baba72c..0000000 --- a/config.toml +++ /dev/null @@ -1,19 +0,0 @@ -[settings] -allow_nsfw = false -theme = "light" -times_to_run = 1 -opacity = 0.9 -storymode = false - -[reddit.creds] -client_id = "adsfadsfasdf" -client_secret = "adsfasdfadsfasdfasdf" -username = "asdfasdfadsf" -password = "asdfasdfadsf" -2fa = false - -[reddit.thread] -random = false -subreddit = "asdfasdfadsfadsfadfs" -post_id = "asdfasdfasdf" -max_comment_length = 500 diff --git a/main.py b/main.py index e362aad..b7fdc5c 100755 --- a/main.py +++ b/main.py @@ -1,12 +1,11 @@ #!/usr/bin/env python import math from subprocess import Popen -from os import getenv, name -from dotenv import load_dotenv +from os import name from reddit.subreddit import get_subreddit_threads from utils.cleanup import cleanup from utils.console import print_markdown, print_step -from utils.checker import check_env +from utils.tomlchecker import check_toml # from utils.checker import envUpdate from video_creation.background import download_background, chop_background_video @@ -31,6 +30,7 @@ print_markdown( ) print_step(f"You are using V{VERSION} of the bot") + def main(POST_ID=None): cleanup() reddit_object = get_subreddit_threads(POST_ID) @@ -45,25 +45,24 @@ def main(POST_ID=None): def run_many(times): for x in range(1, times + 1): print_step( - f'on the {x}{("st" if x == 1 else ("nd" if x == 2 else ("rd" if x == 3 else "th")))} iteration of {times}' + f'on the {x}{("th", "st", "nd", "rd", "th", "th", "th", "th","th", "th")[x%10]} iteration of {times}' ) # correct 1st 2nd 3rd 4th 5th.... main() Popen("cls" if name == "nt" else "clear", shell=True).wait() if __name__ == "__main__": - if check_env() is not True: - exit() - load_dotenv() + config = check_toml(".config.template.toml", "config.toml") + config is False and exit() try: - if getenv("TIMES_TO_RUN") and isinstance(int(getenv("TIMES_TO_RUN")), int): - run_many(int(getenv("TIMES_TO_RUN"))) + if config["settings"]["times_to_run"]: + run_many(config["settings"]["times_to_run"]) - elif len(getenv("POST_ID", "").split("+")) > 1: - for index, post_id in enumerate(getenv("POST_ID", "").split("+")): + elif len(config["reddit"]["thread"]["post_id"].split("+")) > 1: + for index, post_id in enumerate(config["reddit"]["thread"]["post_id"].split("+")): index += 1 print_step( - f'on the {index}{("st" if index == 1 else ("nd" if index == 2 else ("rd" if index == 3 else "th")))} post of {len(getenv("POST_ID", "").split("+"))}' + f'on the {index}{("st" if index%10 == 1 else ("nd" if index%10 == 2 else ("rd" if index%10 == 3 else "th")))} post of {len(config["reddit"]["thread"]["post_id"].split("+"))}' ) main(post_id) Popen("cls" if name == "nt" else "clear", shell=True).wait() diff --git a/utils/console.py b/utils/console.py index fdab5d5..46396f2 100644 --- a/utils/console.py +++ b/utils/console.py @@ -23,14 +23,13 @@ def print_step(text): panel = Panel(Text(text, justify="left")) console.print(panel) - + def print_table(items): """Prints items in a table.""" console.print(Columns([Panel(f"[yellow]{item}", expand=True) for item in items])) - def print_substep(text, style=""): """Prints a rich info message without the panelling.""" console.print(text, style=style) @@ -80,8 +79,7 @@ def handle_input( continue elif match != "" and re.match(match, user_input) is None: console.print( - "[red]" + err_message + - "\nAre you absolutely sure it's correct?(y/n)" + "[red]" + err_message + "\nAre you absolutely sure it's correct?(y/n)" ) if input().casefold().startswith("y"): break @@ -99,13 +97,25 @@ def handle_input( while True: console.print(message, end="") user_input = input("").strip() - if user_input not in options: - console.print( - "[red bold]" - + err_message - + "\nValid options are: " - + ", ".join(map(str, options)) - + "." - ) - continue - return user_input + if check_type is not False: + try: + isinstance(eval(user_input), check_type) + return check_type(user_input) + except: + console.print( + "[red bold]" + + err_message + + "\nValid options are: " + + ", ".join(map(str, options)) + + "." + ) + continue + if user_input in options: + return user_input + console.print( + "[red bold]" + + err_message + + "\nValid options are: " + + ", ".join(map(str, options)) + + "." + ) diff --git a/utils/tomlchecker.py b/utils/tomlchecker.py index 85f6a58..c4e8c5a 100755 --- a/utils/tomlchecker.py +++ b/utils/tomlchecker.py @@ -5,14 +5,12 @@ from rich import pretty from rich.console import Console import re -# from utils.console import handle_input -from console import handle_input +from utils.console import handle_input - -console = Console() +# from console import handle_input -printed = False +console = Console() def crawl(obj: dict, func=lambda x, y: print(x, y, end="\n"), path: list = []): @@ -24,124 +22,50 @@ def crawl(obj: dict, func=lambda x, y: print(x, y, end="\n"), path: list = []): def check(value, checks, name): - global printed - if printed is False: - console.print( - """\ -[blue bold]############################### -# # -# Checking TOML configuration # -# # -############################### -If you see any prompts, that means that you have unset/incorrectly set variables, please input the correct values.\ -""" - ) - printed = True - if "type" in checks: + + incorrect = False + if value == {}: + incorrect = True + if not incorrect and "type" in checks: try: value = eval(checks["type"])(value) except: - value = handle_input( - message=( - ( - ("[blue]Example: " + str(checks["example"]) + "\n") - if "example" in checks - else "" - ) - + "[red]" - + ("Non-optional ", "Optional ")[ - "optional" in checks and checks["optional"] is True - ] - ) - + " [#C0CAF5 bold]" - + str(name) - + "[#F7768E bold]=", - check_type=eval(checks["type"]), - extra_info=checks["explanation"] if "explanation" in checks else "", - default=checks["default"] if "default" in checks else NotImplemented, - match=checks["regex"] if "regex" in checks else "", - err_message=checks["input_error"] if "input_error" in checks else "Incorrect input", - nmin=checks["nmin"] if "nmin" in checks else None, - nmax=checks["nmax"] if "nmax" in checks else None, - oob_error=checks["oob_error"] - if "oob_error" in checks - else "Input out of bounds(Value too high/low/long/short)", - ) + incorrect = True if ( - "options" in checks and value not in checks["options"] + not incorrect and "options" in checks and value not in checks["options"] ): # FAILSTATE Value is not one of the options - value = handle_input( - message=( - (("[blue]Example: " + str(checks["example"]) + "\n") if "example" in checks else "") - + "[red]" - + ("Non-optional ", "Optional ")[ - "optional" in checks and checks["optional"] is True - ] - ) - + "[#C0CAF5 bold]" - + str(name) - + "[#F7768E bold]=", - extra_info=checks["explanation"] if "explanation" in checks else "", - err_message=checks["input_error"] if "input_error" in checks else "Incorrect input", - default=checks["default"] if "default" in checks else NotImplemented, - options=checks["options"], + incorrect = True + if ( + not incorrect + and "regex" in checks + and ( + (isinstance(value, str) and re.match(checks["regex"], value) is None) + or not isinstance(value, str) ) - if "regex" in checks and ( - (isinstance(value, str) and re.match(checks["regex"], value) is None) - or not isinstance(value, str) ): # FAILSTATE Value doesn't match regex, or has regex but is not a string. - value = handle_input( - message=( - (("[blue]Example: " + str(checks["example"]) + "\n") if "example" in checks else "") - + "[red]" - + ("Non-optional ", "Optional ")[ - "optional" in checks and checks["optional"] is True - ] - ) - + "[#C0CAF5 bold]" - + str(name) - + "[#F7768E bold]=", - extra_info=checks["explanation"] if "explanation" in checks else "", - match=checks["regex"], - err_message=checks["input_error"] if "input_error" in checks else "Incorrect input", - default=checks["default"] if "default" in checks else NotImplemented, - nmin=checks["nmin"] if "nmin" in checks else None, - nmax=checks["nmax"] if "nmax" in checks else None, - oob_error=checks["oob_error"] - if "oob_error" in checks - else "Input out of bounds(Value too high/low/long/short)", - ) + incorrect = True - if not hasattr(value, "__iter__") and ( - ("nmin" in checks and checks["nmin"] is not None and value < checks["nmin"]) - or ("nmax" in checks and checks["nmax"] is not None and value > checks["nmax"]) + if ( + not incorrect + and not hasattr(value, "__iter__") + and ( + ("nmin" in checks and checks["nmin"] is not None and value < checks["nmin"]) + or ("nmax" in checks and checks["nmax"] is not None and value > checks["nmax"]) + ) ): - value = handle_input( - message=( - (("[blue]Example: " + str(checks["example"]) + "\n") if "example" in checks else "") - + "[red]" - + ("Non-optional ", "Optional ")[ - "optional" in checks and checks["optional"] is True - ] - ) - + "[#C0CAF5 bold]" - + str(name) - + "[#F7768E bold]=", - extra_info=checks["explanation"] if "explanation" in checks else "", - default=checks["default"] if "default" in checks else NotImplemented, - match=checks["regex"] if "regex" in checks else "", - err_message=checks["input_error"] if "input_error" in checks else "Incorrect input", - nmin=checks["nmin"] if "nmin" in checks else None, - nmax=checks["nmax"] if "nmax" in checks else None, - oob_error=checks["oob_error"] - if "oob_error" in checks - else "Input out of bounds(Value too high/low/long/short)", + incorrect = True + if ( + not incorrect + and hasattr(value, "__iter__") + and ( + ("nmin" in checks and checks["nmin"] is not None and len(value) < checks["nmin"]) + or ("nmax" in checks and checks["nmax"] is not None and len(value) > checks["nmax"]) ) - if hasattr(value, "__iter__") and ( - ("nmin" in checks and checks["nmin"] is not None and len(value) < checks["nmin"]) - or ("nmax" in checks and checks["nmax"] is not None and len(value) > checks["nmax"]) ): + incorrect = True + + if incorrect: value = handle_input( message=( (("[blue]Example: " + str(checks["example"]) + "\n") if "example" in checks else "") @@ -154,6 +78,7 @@ If you see any prompts, that means that you have unset/incorrectly set variables + str(name) + "[#F7768E bold]=", extra_info=checks["explanation"] if "explanation" in checks else "", + check_type=eval(checks["type"]) if "type" in checks else False, default=checks["default"] if "default" in checks else NotImplemented, match=checks["regex"] if "regex" in checks else "", err_message=checks["input_error"] if "input_error" in checks else "Incorrect input", @@ -162,28 +87,7 @@ If you see any prompts, that means that you have unset/incorrectly set variables oob_error=checks["oob_error"] if "oob_error" in checks else "Input out of bounds(Value too high/low/long/short)", - ) - if value == {}: - handle_input( - message=( - (("[blue]Example: " + str(checks["example"]) + "\n") if "example" in checks else "") - + "[red]" - + ("Non-optional ", "Optional ")[ - "optional" in checks and checks["optional"] is True - ] - ) - + "[#C0CAF5 bold]" - + str(name) - + "[#F7768E bold]=", - extra_info=checks["explanation"] if "explanation" in checks else "", - default=checks["default"] if "default" in checks else NotImplemented, - match=checks["regex"] if "regex" in checks else "", - err_message=checks["input_error"] if "input_error" in checks else "Incorrect input", - nmin=checks["nmin"] if "nmin" in checks else None, - nmax=checks["nmax"] if "nmax" in checks else None, - oob_error=checks["oob_error"] - if "oob_error" in checks - else "Input out of bounds(Value too high/low/long/short)", + options=checks["options"] if "options" in checks else None, ) return value @@ -244,11 +148,22 @@ Creating it now.""" f"[red bold]Failed to write to {config_file}. Giving up.\nSuggestion: check the folder's permissions for the user." ) return False + + console.print( + """\ +[blue bold]############################### +# # +# Checking TOML configuration # +# # +############################### +If you see any prompts, that means that you have unset/incorrectly set variables, please input the correct values.\ +""" + ) crawl(template, check_vars) # pretty.pprint(config) with open(config_file, "w") as f: toml.dump(config, f) - return True + return config if __name__ == "__main__": From 4d9ce59f24d516a8414b13d1ec4af351fd4d3da9 Mon Sep 17 00:00:00 2001 From: CordlessCoder Date: Tue, 5 Jul 2022 23:25:15 +0300 Subject: [PATCH 034/126] feat: correct optional variable behavour --- .config.template.toml | 10 ++-- .env.template | 86 --------------------------- main.py | 2 +- utils/console.py | 5 ++ utils/{tomlchecker.py => settings.py} | 16 +++-- 5 files changed, 23 insertions(+), 96 deletions(-) delete mode 100644 .env.template rename utils/{tomlchecker.py => settings.py} (91%) diff --git a/.config.template.toml b/.config.template.toml index 1c5ee43..d12d30b 100644 --- a/.config.template.toml +++ b/.config.template.toml @@ -15,21 +15,21 @@ random = { optional = true, options = [ false, ], default = false, type = "bool", explanation = "If set to no, it will ask you a thread link to extract the thread, if yes it will randomize it. Default: 'False'", example = "True" } subreddit = { optional = false, regex = "[_0-9a-zA-Z]+$", nmin = 3, nmax = 21, explanation = "what subreddit to pull posts from, the name of the sub, not the URL", example = "AskReddit", oob_error = "A subreddit name HAS to be between 3 and 20 characters" } -post_id = { optional = false, regex = "^((?!://|://).)*$", explanation = "Used if you want to use a specific post.", example = "urdtfx" } +post_id = { optional = true, regex = "^((?!://|://).)*$", explanation = "Used if you want to use a specific post.", example = "urdtfx" } max_comment_length = { default = 500, optional = false, nmin = 10, nmax = 10000, type = "int", explanation = "max number of characters a comment can have. default is 500", example = 500, oob_error = "the max comment length should be between 10 and 10000" } [settings] -allow_nsfw = { optional = true, type = "bool", default = false, example = false, options = [ +allow_nsfw = { optional = false, type = "bool", default = false, example = false, options = [ true, false, ], explanation = "Whether to allow NSFW content, True or False" } -theme = { optional = true, default = "light", example = "dark", options = [ +theme = { optional = false, default = "light", example = "dark", options = [ "dark", "light", ], explanation = "sets the Reddit theme, either LIGHT or DARK" } -times_to_run = { optional = true, default = 1, example = 2, explanation = "used if you want to run multiple times. set to an int e.g. 4 or 29 or 1", type = "int", nmin = 1, oob_error = "It's very hard to run something less than once." } -opacity = { optional = true, default = 0.9, example = 0.8, explanation = "Sets the opacity of the comments when overlayed over the background", type = "float", nmin = 0, nmax = 1, oob_error = "The opacity HAS to be between 0 and 1", input_error = "The opacity HAS to be a decimal number between 0 and 1" } +times_to_run = { optional = false, default = 1, example = 2, explanation = "used if you want to run multiple times. set to an int e.g. 4 or 29 or 1", type = "int", nmin = 1, oob_error = "It's very hard to run something less than once." } +opacity = { optional = false, default = 0.9, example = 0.8, explanation = "Sets the opacity of the comments when overlayed over the background", type = "float", nmin = 0, nmax = 1, oob_error = "The opacity HAS to be between 0 and 1", input_error = "The opacity HAS to be a decimal number between 0 and 1" } storymode = { optional = true, type = "bool", default = false, example = false, options = [ true, false, diff --git a/.env.template b/.env.template deleted file mode 100644 index 77f2acf..0000000 --- a/.env.template +++ /dev/null @@ -1,86 +0,0 @@ - -REDDIT_CLIENT_ID="" #fFAGRNJru1FTz70BzhT3Zg -#EXPLANATION the ID of your Reddit app of SCRIPT type -#RANGE 12:30 -#MATCH_REGEX [-a-zA-Z0-9._~+/]+=*$ -#OOB_ERROR The ID should be over 12 and under 30 characters, double check your input. - -REDDIT_CLIENT_SECRET="" #fFAGRNJru1FTz70BzhT3Zg -#EXPLANATION the SECRET of your Reddit app of SCRIPT type -#RANGE 20:40 -#MATCH_REGEX [-a-zA-Z0-9._~+/]+=*$ -#OOB_ERROR The secret should be over 20 and under 40 characters, double check your input. - -REDDIT_USERNAME="" #asdfghjkl -#EXPLANATION the username of your reddit account -#RANGE 3:20 -#MATCH_REGEX [-_0-9a-zA-Z]+$ -#OOB_ERROR A username HAS to be between 3 and 20 characters - -REDDIT_PASSWORD="" #fFAGRNJru1FTz70BzhT3Zg -#EXPLANATION the password of your reddit account -#RANGE 8:None -#OOB_ERROR Password too short - -#OPTIONAL -RANDOM_THREAD="no" -# If set to no, it will ask you a thread link to extract the thread, if yes it will randomize it. Default: "no" - -REDDIT_2FA="" #no -#MATCH_REGEX ^(yes|no) -#EXPLANATION Whether you have Reddit 2FA enabled, Valid options are "yes" and "no" - -SUBREDDIT="AskReddit" -#EXPLANATION what subreddit to pull posts from, the name of the sub, not the URL -#RANGE 3:20 -#MATCH_REGEX [_0-9a-zA-Z]+$ -#OOB_ERROR A subreddit name HAS to be between 3 and 20 characters - -ALLOW_NSFW="False" -#EXPLANATION Whether to allow NSFW content, True or False -#MATCH_REGEX ^(True|False)$ - -POST_ID="" -#MATCH_REGEX ^((?!://|://).)*$ -#EXPLANATION Used if you want to use a specific post. example of one is urdtfx - -THEME="LIGHT" #dark -#EXPLANATION sets the Reddit theme, either LIGHT or DARK -#MATCH_REGEX ^(dark|light|DARK|LIGHT)$ - -TIMES_TO_RUN="" #2 -#EXPLANATION used if you want to run multiple times. set to an int e.g. 4 or 29 and leave blank for 1 - -MAX_COMMENT_LENGTH="500" #500 -#EXPLANATION max number of characters a comment can have. default is 500 -#RANGE 0:10000 -#MATCH_TYPE int -#OOB_ERROR the max comment length should be between 0 and 10000 - -OPACITY="1" #.8 -#EXPLANATION Sets the opacity of the comments when overlayed over the background -#RANGE 0:1 -#MATCH_TYPE float -#OOB_ERROR The opacity HAS to be between 0 and 1 - -# If you want to translate the comments to another language, set the language code here. -# If empty, no translation will be done. -POSTLANG="" -#EXPLANATION Activates the translation feature, set the language code for translate or leave blank - -TTSCHOICE="Polly" -#EXPLANATION the backend used for TTS. Without anything specified, the user will be prompted to choose one. -# IMPORTANT NOTE: if you use translate, you need to set this to googletranslate or tiktok and use custom voice in your language - -STREAMLABS_VOICE="Joanna" -#EXPLANATION Sets the voice for the Streamlabs Polly TTS Engine. Check the file for more information on different voices. - -AWS_VOICE="Joanna" -#EXPLANATION Sets the voice for the AWS Polly TTS Engine. Check the file for more information on different voices. - -TIKTOK_VOICE="en_us_006" -#EXPLANATION Sets the voice for the TikTok TTS Engine. Check the file for more information on different voices. - -#OPTIONAL -STORYMODE="False" -# IN-PROGRESS - not yet implemented diff --git a/main.py b/main.py index b7fdc5c..8fc3b59 100755 --- a/main.py +++ b/main.py @@ -5,7 +5,7 @@ from os import name from reddit.subreddit import get_subreddit_threads from utils.cleanup import cleanup from utils.console import print_markdown, print_step -from utils.tomlchecker import check_toml +from utils.settings import check_toml # from utils.checker import envUpdate from video_creation.background import download_background, chop_background_video diff --git a/utils/console.py b/utils/console.py index 46396f2..310247d 100644 --- a/utils/console.py +++ b/utils/console.py @@ -46,7 +46,12 @@ def handle_input( extra_info="", options: list = None, default=NotImplemented, + optional=False, ): + if optional: + console.print(message + "\n[green]This is an optional value. Do you want to skip it? (y/n)") + if input().casefold().startswith("y"): + return None if default is not NotImplemented: console.print( "[green]" diff --git a/utils/tomlchecker.py b/utils/settings.py similarity index 91% rename from utils/tomlchecker.py rename to utils/settings.py index c4e8c5a..927c500 100755 --- a/utils/tomlchecker.py +++ b/utils/settings.py @@ -1,7 +1,6 @@ #!/usr/bin/env python # import os import toml -from rich import pretty from rich.console import Console import re @@ -25,6 +24,8 @@ def check(value, checks, name): incorrect = False if value == {}: + if skip_opt and "optional" in checks and checks["optional"] is True: + return None incorrect = True if not incorrect and "type" in checks: try: @@ -88,6 +89,7 @@ def check(value, checks, name): if "oob_error" in checks else "Input out of bounds(Value too high/low/long/short)", options=checks["options"] if "options" in checks else None, + optional=checks["optional"] if "optional" in checks else False, ) return value @@ -103,10 +105,13 @@ def crawl_and_check(obj: dict, path: list, checks: dict = {}, name=""): def check_vars(path, checks): global config + global skip_opt + skip_opt = "skip_opt" in config crawl_and_check(config, path, checks) -def check_toml(template_file, config_file) -> bool: +def check_toml(template_file, config_file) -> (bool, dict): + global config try: template = toml.load(template_file) except Exception as error: @@ -115,7 +120,6 @@ def check_toml(template_file, config_file) -> bool: ) return False try: - global config config = toml.load(config_file) except (toml.TomlDecodeError): console.print( @@ -160,7 +164,7 @@ If you see any prompts, that means that you have unset/incorrectly set variables """ ) crawl(template, check_vars) - # pretty.pprint(config) + config["skip_opt"] = True with open(config_file, "w") as f: toml.dump(config, f) return config @@ -168,3 +172,7 @@ If you see any prompts, that means that you have unset/incorrectly set variables if __name__ == "__main__": check_toml(".config.template.toml", "config.toml") + +if __name__ == "__main__": + check_toml(".config.template.toml", "config.toml") + check_toml(".config.template.toml", "config.toml") From 4c1c80ffd7e2be02b74443cb04cb3817b9ee28e4 Mon Sep 17 00:00:00 2001 From: CordlessCoder Date: Tue, 5 Jul 2022 23:26:07 +0300 Subject: [PATCH 035/126] chore: removed unused files --- utils/checker.py | 193 ----------------------------------------------- utils/config.py | 46 ----------- 2 files changed, 239 deletions(-) delete mode 100755 utils/checker.py delete mode 100644 utils/config.py diff --git a/utils/checker.py b/utils/checker.py deleted file mode 100755 index 791a376..0000000 --- a/utils/checker.py +++ /dev/null @@ -1,193 +0,0 @@ -#!/usr/bin/env python -import os -from rich.console import Console -from rich.table import Table -from rich import box -import re -import dotenv -from utils.console import handle_input - -console = Console() - - -def check_env() -> bool: - """Checks to see what's been put in .env - - Returns: - bool: Whether or not everything was put in properly - """ - if not os.path.exists(".env.template"): - console.print("[red]Couldn't find .env.template. Unable to check variables.") - return True - if not os.path.exists(".env"): - console.print("[red]Couldn't find the .env file, creating one now.") - with open(".env", "x", encoding="utf-8") as file: - file.write("") - success = True - with open(".env.template", "r", encoding="utf-8") as template: - # req_envs = [env.split("=")[0] for env in template.readlines() if "=" in env] - matching = {} - explanations = {} - bounds = {} - types = {} - oob_errors = {} - examples = {} - req_envs = [] - var_optional = False - for line in template.readlines(): - if line.startswith("#") is not True and "=" in line and var_optional is not True: - req_envs.append(line.split("=")[0]) - if "#" in line: - examples[line.split("=")[0]] = "#".join(line.split("#")[1:]).strip() - elif "#OPTIONAL" in line: - var_optional = True - elif line.startswith("#MATCH_REGEX "): - matching[req_envs[-1]] = line.removeprefix("#MATCH_REGEX ")[:-1] - var_optional = False - elif line.startswith("#OOB_ERROR "): - oob_errors[req_envs[-1]] = line.removeprefix("#OOB_ERROR ")[:-1] - var_optional = False - elif line.startswith("#RANGE "): - bounds[req_envs[-1]] = tuple( - map( - lambda x: float(x) if x != "None" else None, - line.removeprefix("#RANGE ")[:-1].split(":"), - ) - ) - var_optional = False - elif line.startswith("#MATCH_TYPE "): - types[req_envs[-1]] = eval(line.removeprefix("#MATCH_TYPE ")[:-1].split()[0]) - var_optional = False - elif line.startswith("#EXPLANATION "): - explanations[req_envs[-1]] = line.removeprefix("#EXPLANATION ")[:-1] - var_optional = False - else: - var_optional = False - missing = set() - incorrect = set() - dotenv.load_dotenv() - for env in req_envs: - value = os.getenv(env) - if value is None: - missing.add(env) - continue - if env in matching.keys(): - re.match(matching[env], value) is None and incorrect.add(env) - if env in bounds.keys() and env not in types.keys(): - len(value) >= bounds[env][0] or ( - len(bounds[env]) > 1 and bounds[env][1] >= len(value) - ) or incorrect.add(env) - continue - if env in types.keys(): - try: - temp = types[env](value) - if env in bounds.keys(): - (bounds[env][0] <= temp or incorrect.add(env)) and len(bounds[env]) > 1 and ( - bounds[env][1] >= temp or incorrect.add(env) - ) - except ValueError: - incorrect.add(env) - - if len(missing): - table = Table( - title="Missing variables", - highlight=True, - show_lines=True, - box=box.ROUNDED, - border_style="#414868", - header_style="#C0CAF5 bold", - title_justify="left", - title_style="#C0CAF5 bold", - ) - table.add_column("Variable", justify="left", style="#7AA2F7 bold", no_wrap=True) - table.add_column("Explanation", justify="left", style="#BB9AF7", no_wrap=False) - table.add_column("Example", justify="center", style="#F7768E", no_wrap=True) - table.add_column("Min", justify="right", style="#F7768E", no_wrap=True) - table.add_column("Max", justify="left", style="#F7768E", no_wrap=True) - for env in missing: - table.add_row( - env, - explanations[env] if env in explanations.keys() else "No explanation given", - examples[env] if env in examples.keys() else "", - str(bounds[env][0]) if env in bounds.keys() and bounds[env][1] is not None else "", - str(bounds[env][1]) - if env in bounds.keys() and len(bounds[env]) > 1 and bounds[env][1] is not None - else "", - ) - console.print(table) - success = False - if len(incorrect): - table = Table( - title="Incorrect variables", - highlight=True, - show_lines=True, - box=box.ROUNDED, - border_style="#414868", - header_style="#C0CAF5 bold", - title_justify="left", - title_style="#C0CAF5 bold", - ) - table.add_column("Variable", justify="left", style="#7AA2F7 bold", no_wrap=True) - table.add_column("Current value", justify="left", style="#F7768E", no_wrap=False) - table.add_column("Explanation", justify="left", style="#BB9AF7", no_wrap=False) - table.add_column("Example", justify="center", style="#F7768E", no_wrap=True) - table.add_column("Min", justify="right", style="#F7768E", no_wrap=True) - table.add_column("Max", justify="left", style="#F7768E", no_wrap=True) - for env in incorrect: - table.add_row( - env, - os.getenv(env), - explanations[env] if env in explanations.keys() else "No explanation given", - str(types[env].__name__) if env in types.keys() else "str", - str(bounds[env][0]) if env in bounds.keys() else "None", - str(bounds[env][1]) if env in bounds.keys() and len(bounds[env]) > 1 else "None", - ) - missing.add(env) - console.print(table) - success = False - if success is True: - return True - console.print( - "[green]Do you want to automatically overwrite incorrect variables and add the missing variables? (y/n)" - ) - if not input().casefold().startswith("y"): - console.print("[red]Aborting: Unresolved missing variables") - return False - if len(incorrect): - with open(".env", "r+", encoding="utf-8") as env_file: - lines = [] - for line in env_file.readlines(): - line.split("=")[0].strip() not in incorrect and lines.append(line) - env_file.seek(0) - env_file.write("\n".join(lines)) - env_file.truncate() - console.print("[green]Successfully removed incorrectly set variables from .env") - with open(".env", "a", encoding="utf-8") as env_file: - for env in missing: - env_file.write( - env - + "=" - + ('"' if env not in types.keys() else "") - + str( - handle_input( - "[#F7768E bold]" + env + "[#C0CAF5 bold]=", - types[env] if env in types.keys() else False, - matching[env] if env in matching.keys() else ".*", - explanations[env] - if env in explanations.keys() - else "Incorrect input. Try again.", - bounds[env][0] if env in bounds.keys() else None, - bounds[env][1] if env in bounds.keys() and len(bounds[env]) > 1 else None, - oob_errors[env] if env in oob_errors.keys() else "Input too long/short.", - extra_info="[#C0CAF5 bold]⮶ " - + (explanations[env] if env in explanations.keys() else "No info available"), - ) - ) - + ('"' if env not in types.keys() else "") - + "\n" - ) - return True - - -if __name__ == "__main__": - check_env() diff --git a/utils/config.py b/utils/config.py deleted file mode 100644 index 29cbb79..0000000 --- a/utils/config.py +++ /dev/null @@ -1,46 +0,0 @@ -# write a class that takes .env file and parses it into a dictionary -from dotenv import dotenv_values - -DEFAULTS = { - "SUBREDDIT": "AskReddit", - "ALLOW_NSFW": "False", - "POST_ID": "", - "THEME": "DARK", - "REDDIT_2FA": "no", - "TIMES_TO_RUN": "", - "MAX_COMMENT_LENGTH": "500", - "OPACITY": "1", - "VOICE": "en_us_001", - "STORYMODE": "False", -} - - -class Config: - def __init__(self): - self.raw = dotenv_values("../.env") - self.load_attrs() - - def __getattr__(self, attr): # code completion for attributes fix. - return getattr(self, attr) - - def load_attrs(self): - for key, value in self.raw.items(): - self.add_attr(key, value) - - def add_attr(self, key, value): - if value is None or value == "": - setattr(self, key, DEFAULTS[key]) - else: - setattr(self, key, str(value)) - - -config = Config() - -print(config.SUBREDDIT) -# def temp(): -# root = '' -# if isinstance(root, praw.models.Submission): -# root_type = 'submission' -# elif isinstance(root, praw.models.Comment): -# root_type = 'comment' -# From 75c731ce5cc05e56d3edf3b28b1f64ad51bfc726 Mon Sep 17 00:00:00 2001 From: Callum Leslie Date: Tue, 5 Jul 2022 21:01:39 +0100 Subject: [PATCH 036/126] fix: spelling error --- .config.template.toml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.config.template.toml b/.config.template.toml index d12d30b..0ea6da2 100644 --- a/.config.template.toml +++ b/.config.template.toml @@ -6,7 +6,7 @@ password = { optional = false, nmin = 8, explanation = "the password of your red 2fa = { optional = true, type = "bool", options = [ true, false, -], default = false, explanation = "Whether you have Reddit 2FA enabled, Valid options are True and Talse", example = true } +], default = false, explanation = "Whether you have Reddit 2FA enabled, Valid options are True and False", example = true } [reddit.thread] From d2d593371bcde615469ed68c142eebd5afab6935 Mon Sep 17 00:00:00 2001 From: Callum Leslie Date: Tue, 5 Jul 2022 21:02:31 +0100 Subject: [PATCH 037/126] refactor: rename tomlchecker and move to global settings variable --- utils/settings.py | 9 ++++----- 1 file changed, 4 insertions(+), 5 deletions(-) diff --git a/utils/settings.py b/utils/settings.py index 927c500..d674974 100755 --- a/utils/settings.py +++ b/utils/settings.py @@ -4,6 +4,8 @@ import toml from rich.console import Console import re +from typing import Tuple, Dict + from utils.console import handle_input # from console import handle_input @@ -110,8 +112,9 @@ def check_vars(path, checks): crawl_and_check(config, path, checks) -def check_toml(template_file, config_file) -> (bool, dict): +def check_toml(template_file, config_file) -> Tuple[bool, Dict]: global config + config = None try: template = toml.load(template_file) except Exception as error: @@ -172,7 +175,3 @@ If you see any prompts, that means that you have unset/incorrectly set variables if __name__ == "__main__": check_toml(".config.template.toml", "config.toml") - -if __name__ == "__main__": - check_toml(".config.template.toml", "config.toml") - check_toml(".config.template.toml", "config.toml") From a110f49e4494172b0f4d0a35a978dccdb94c8063 Mon Sep 17 00:00:00 2001 From: Callum Leslie Date: Tue, 5 Jul 2022 21:04:01 +0100 Subject: [PATCH 038/126] refactor: use new settings option --- main.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/main.py b/main.py index 8fc3b59..7b7c726 100755 --- a/main.py +++ b/main.py @@ -5,7 +5,7 @@ from os import name from reddit.subreddit import get_subreddit_threads from utils.cleanup import cleanup from utils.console import print_markdown, print_step -from utils.settings import check_toml +from utils import settings # from utils.checker import envUpdate from video_creation.background import download_background, chop_background_video @@ -52,7 +52,7 @@ def run_many(times): if __name__ == "__main__": - config = check_toml(".config.template.toml", "config.toml") + config = settings.check_toml(".config.template.toml", "config.toml") config is False and exit() try: if config["settings"]["times_to_run"]: From b1d27a3b848038161616e4e58d41debc1702704b Mon Sep 17 00:00:00 2001 From: Callum Leslie Date: Tue, 5 Jul 2022 21:13:59 +0100 Subject: [PATCH 039/126] refactor: subreddit.py uses toml config --- utils/subreddit.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/utils/subreddit.py b/utils/subreddit.py index f6ca686..a5d7aa6 100644 --- a/utils/subreddit.py +++ b/utils/subreddit.py @@ -1,5 +1,5 @@ import json -from os import getenv +from utils import settings from utils.console import print_substep @@ -22,7 +22,7 @@ def get_subreddit_undone(submissions: list, subreddit): continue if submission.over_18: try: - if getenv("ALLOW_NSFW").casefold() == "false": + if settings.config["settings"]["allow_nsfw"] == "false": print_substep("NSFW Post Detected. Skipping...") continue except AttributeError: From bc631c57afde288540ddd0fd29671e2ae7eb836d Mon Sep 17 00:00:00 2001 From: Callum Leslie Date: Tue, 5 Jul 2022 21:33:18 +0100 Subject: [PATCH 040/126] refactor: reddit/subreddit.py uses toml config --- reddit/subreddit.py | 29 ++++++++++++++--------------- 1 file changed, 14 insertions(+), 15 deletions(-) diff --git a/reddit/subreddit.py b/reddit/subreddit.py index e1f8940..5eed982 100644 --- a/reddit/subreddit.py +++ b/reddit/subreddit.py @@ -1,6 +1,6 @@ import re -from os import getenv +from utils import settings import praw from praw.models import MoreComments @@ -17,20 +17,20 @@ def get_subreddit_threads(POST_ID: str): print_substep("Logging into Reddit.") content = {} - if str(getenv("REDDIT_2FA")).casefold() == "yes": + if settings.config["reddit"]["creds"]["2fa"] == "true": print("\nEnter your two-factor authentication code from your authenticator app.\n") code = input("> ") print() - pw = getenv("REDDIT_PASSWORD") + pw = settings.config["reddit"]["creds"]["password"] passkey = f"{pw}:{code}" else: - passkey = getenv("REDDIT_PASSWORD") - username = getenv("REDDIT_USERNAME") + passkey = settings.config["reddit"]["creds"]["password"] + username = settings.config["reddit"]["creds"]["username"] if username.casefold().startswith("u/"): username = username[2:] reddit = praw.Reddit( - client_id=getenv("REDDIT_CLIENT_ID"), - client_secret=getenv("REDDIT_CLIENT_SECRET"), + client_id= settings.config["reddit"]["creds"]["client_id"], + client_secret=settings.config["reddit"]["creds"]["client_secret"], user_agent="Accessing Reddit threads", username=username, passkey=passkey, @@ -39,9 +39,7 @@ def get_subreddit_threads(POST_ID: str): # Ask user for subreddit input print_step("Getting subreddit threads...") - if not getenv( - "SUBREDDIT" - ): # note to user. you can have multiple subreddits via reddit.subreddit("redditdev+learnpython") + if not settings.config["reddit"]["thread"]["subreddit"]: # note to user. you can have multiple subreddits via reddit.subreddit("redditdev+learnpython") try: subreddit = reddit.subreddit( re.sub(r"r\/", "", input("What subreddit would you like to pull from? ")) @@ -51,8 +49,9 @@ def get_subreddit_threads(POST_ID: str): subreddit = reddit.subreddit("askreddit") print_substep("Subreddit not defined. Using AskReddit.") else: - print_substep(f"Using subreddit: r/{getenv('SUBREDDIT')} from environment variable config") - subreddit_choice = getenv("SUBREDDIT") + sub = settings.config["reddit"]["thread"]["subreddit"] + print_substep(f"Using subreddit: r/{sub} from TOML config") + subreddit_choice = sub if subreddit_choice.casefold().startswith("r/"): # removes the r/ from the input subreddit_choice = subreddit_choice[2:] subreddit = reddit.subreddit( @@ -61,8 +60,8 @@ def get_subreddit_threads(POST_ID: str): if POST_ID: # would only be called if there are multiple queued posts submission = reddit.submission(id=POST_ID) - elif getenv("POST_ID") and len(getenv("POST_ID").split("+")) == 1: - submission = reddit.submission(id=getenv("POST_ID")) + elif settings.config["reddit"]["thread"]["post_id"] and len(settings.config["reddit"]["thread"]["post_id"].split("+")) == 1: + submission = reddit.submission(id=settings.config["reddit"]["thread"]["post_id"]) else: threads = subreddit.hot(limit=25) @@ -91,7 +90,7 @@ def get_subreddit_threads(POST_ID: str): if top_level_comment.body in ["[removed]", "[deleted]"]: continue # # see https://github.com/JasonLovesDoggo/RedditVideoMakerBot/issues/78 if not top_level_comment.stickied: - if len(top_level_comment.body) <= int(getenv("MAX_COMMENT_LENGTH", "500")): + if len(top_level_comment.body) <= int(settings.config["reddit"]["thread"]["max_comment_length"]): if ( top_level_comment.author is not None ): # if errors occur with this change to if not. From 8bc72da9c4e2771e617c3dda1005247ecec927ac Mon Sep 17 00:00:00 2001 From: Callum Leslie Date: Tue, 5 Jul 2022 21:34:09 +0100 Subject: [PATCH 041/126] refactor: utils/videos.py uses new toml config --- utils/videos.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/utils/videos.py b/utils/videos.py index 07659f6..4e248b9 100755 --- a/utils/videos.py +++ b/utils/videos.py @@ -1,11 +1,11 @@ import json import os import time -from os import getenv from typing import Dict from praw.models import Submission +from utils import settings from utils.console import print_step @@ -25,7 +25,7 @@ def check_done( done_videos = json.load(done_vids_raw) for video in done_videos: if video["id"] == str(redditobj): - if getenv("POST_ID"): + if settings.config["reddit"]["thread"]["post_id"]: print_step( "You already have done this video but since it was declared specifically in the .env file the program will continue" ) From 49eec71f0597fab38a53296558bb1be268815bb8 Mon Sep 17 00:00:00 2001 From: CordlessCoder Date: Tue, 5 Jul 2022 23:51:49 +0300 Subject: [PATCH 042/126] refactor: utils/videos.py uses new toml config(fix) --- utils/videos.py | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/utils/videos.py b/utils/videos.py index 4e248b9..6377256 100755 --- a/utils/videos.py +++ b/utils/videos.py @@ -1,5 +1,4 @@ import json -import os import time from typing import Dict @@ -51,7 +50,9 @@ def save_data(filename: str, reddit_title: str, reddit_id: str): payload = { "id": reddit_id, "time": str(int(time.time())), - "background_credit": str(os.getenv("background_credit")), + "background_credit": settings.config["settings"]["background_credit"] + if "background_credit" in settings.config["settings"] + else "", "reddit_title": reddit_title, "filename": filename, } From c1a4b51f410437c7c8180a03a48a9112f314a4a0 Mon Sep 17 00:00:00 2001 From: Callum Leslie Date: Tue, 5 Jul 2022 21:56:04 +0100 Subject: [PATCH 043/126] feat: prepare config template for TTS --- .config.template.toml | 8 +++++--- 1 file changed, 5 insertions(+), 3 deletions(-) diff --git a/.config.template.toml b/.config.template.toml index 0ea6da2..5401118 100644 --- a/.config.template.toml +++ b/.config.template.toml @@ -36,6 +36,8 @@ storymode = { optional = true, type = "bool", default = false, example = false, ] } -# [settings.tts] -# voice = { optional = true, example = "en_us_002", explanation = "sets the voice used by TTS" } -# choice = { optional = true, example = "polly", explanation = "the backend used " } +[settings.tts] +aws_polly_voice = { optional = false, default = "Matthew", example = "Matthew", explanation = "The voice used for AWS Polly" } +streamlabs_polly_voice = { optional = false, default = "Matthew", example = "Matthew", explanation = "The voice used for Streamlabs Polly" } +tiktok_voice = { optional = false, default = "en_us_006", example = "en_us_006", explanation = "The voice used for TikTok TTS" } +choice = { optional = true, example = "streamlabspolly", explanation = "The backend used for TTS generation" } From 19ef71fc9afc78cffbcf515abe2fb45ba2baf894 Mon Sep 17 00:00:00 2001 From: Callum Leslie Date: Tue, 5 Jul 2022 21:56:31 +0100 Subject: [PATCH 044/126] refactor: voices.py uses toml config --- video_creation/voices.py | 9 ++++----- 1 file changed, 4 insertions(+), 5 deletions(-) diff --git a/video_creation/voices.py b/video_creation/voices.py index 5105a10..ace67a4 100644 --- a/video_creation/voices.py +++ b/video_creation/voices.py @@ -1,6 +1,5 @@ #!/usr/bin/env python -import os from typing import Dict, Tuple from rich.console import Console @@ -10,7 +9,7 @@ from TTS.GTTS import GTTS from TTS.streamlabs_polly import StreamlabsPolly from TTS.aws_polly import AWSPolly from TTS.TikTok import TikTok - +from utils import settings from utils.console import print_table, print_step @@ -34,9 +33,9 @@ def save_text_to_mp3(reddit_obj) -> Tuple[int, int]: tuple[int,int]: (total length of the audio, the number of comments audio was generated for) """ - env = os.getenv("TTSCHOICE", "") - if env.casefold() in map(lambda _: _.casefold(), TTSProviders): - text_to_mp3 = TTSEngine(get_case_insensitive_key_value(TTSProviders, env), reddit_obj) + voice = settings.config["tts"]["choice"] + if voice.casefold() in map(lambda _: _.casefold(), TTSProviders): + text_to_mp3 = TTSEngine(get_case_insensitive_key_value(TTSProviders, voice), reddit_obj) else: while True: print_step("Please choose one of the following TTS providers: ") From ac04b665de2c59eac8be3e8a4d571ad8455d8b66 Mon Sep 17 00:00:00 2001 From: Callum Leslie Date: Tue, 5 Jul 2022 22:26:36 +0100 Subject: [PATCH 045/126] refactor: screenshot_downloader.py uses toml config --- video_creation/screenshot_downloader.py | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/video_creation/screenshot_downloader.py b/video_creation/screenshot_downloader.py index e8afc44..c6ed035 100644 --- a/video_creation/screenshot_downloader.py +++ b/video_creation/screenshot_downloader.py @@ -1,9 +1,9 @@ import json import os -from os import getenv + from pathlib import Path from typing import Dict - +from utils import settings from playwright.async_api import async_playwright # pylint: disable=unused-import # do not remove the above line @@ -35,7 +35,7 @@ def download_screenshots_of_reddit_posts(reddit_object: dict, screenshot_num: in browser = p.chromium.launch() context = browser.new_context() - if getenv("THEME").upper() == "DARK": + if settings.config["settings"]["theme"] == "dark": cookie_file = open("./video_creation/data/cookie-dark-mode.json", encoding="utf-8") else: cookie_file = open("./video_creation/data/cookie-light-mode.json", encoding="utf-8") @@ -56,7 +56,7 @@ def download_screenshots_of_reddit_posts(reddit_object: dict, screenshot_num: in # translate code - if getenv("POSTLANG"): + if settings.config["reddit"]["thread"]["post_lang"]: print_substep("Translating post...") texts_in_tl = ts.google(reddit_object["thread_title"], to_language=os.getenv("POSTLANG")) @@ -88,9 +88,9 @@ def download_screenshots_of_reddit_posts(reddit_object: dict, screenshot_num: in # translate code - if getenv("POSTLANG"): + if settings.config["reddit"]["thread"]["post_lang"]: comment_tl = ts.google( - comment["comment_body"], to_language=os.getenv("POSTLANG") + comment["comment_body"], to_language=settings.config["reddit"]["thread"]["post_lang"] ) page.evaluate( '([tl_content, tl_id]) => document.querySelector(`#t1_${tl_id} > div:nth-child(2) > div > div[data-testid="comment"] > div`).textContent = tl_content', From 5a2112b34e93d5a7e9f4ffb79faae1e757df1e9a Mon Sep 17 00:00:00 2001 From: Callum Leslie Date: Tue, 5 Jul 2022 22:38:01 +0100 Subject: [PATCH 046/126] fix: incorrect settings path for tts --- video_creation/voices.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/video_creation/voices.py b/video_creation/voices.py index ace67a4..4224d15 100644 --- a/video_creation/voices.py +++ b/video_creation/voices.py @@ -33,7 +33,7 @@ def save_text_to_mp3(reddit_obj) -> Tuple[int, int]: tuple[int,int]: (total length of the audio, the number of comments audio was generated for) """ - voice = settings.config["tts"]["choice"] + voice = settings.config["settings"]["tts"]["choice"] if voice.casefold() in map(lambda _: _.casefold(), TTSProviders): text_to_mp3 = TTSEngine(get_case_insensitive_key_value(TTSProviders, voice), reddit_obj) else: From 9244cc9b2bc67b8dae6c7290077f961953455746 Mon Sep 17 00:00:00 2001 From: Callum Leslie Date: Tue, 5 Jul 2022 22:38:55 +0100 Subject: [PATCH 047/126] feat: add translation setting to template toml --- .config.template.toml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.config.template.toml b/.config.template.toml index 5401118..7f4a277 100644 --- a/.config.template.toml +++ b/.config.template.toml @@ -17,7 +17,7 @@ random = { optional = true, options = [ subreddit = { optional = false, regex = "[_0-9a-zA-Z]+$", nmin = 3, nmax = 21, explanation = "what subreddit to pull posts from, the name of the sub, not the URL", example = "AskReddit", oob_error = "A subreddit name HAS to be between 3 and 20 characters" } post_id = { optional = true, regex = "^((?!://|://).)*$", explanation = "Used if you want to use a specific post.", example = "urdtfx" } max_comment_length = { default = 500, optional = false, nmin = 10, nmax = 10000, type = "int", explanation = "max number of characters a comment can have. default is 500", example = 500, oob_error = "the max comment length should be between 10 and 10000" } - +post_lang = { optional = true, explanation = "The language you would like to translate to - leave blank for none.", example = "es-cr"} [settings] allow_nsfw = { optional = false, type = "bool", default = false, example = false, options = [ From 0d212868d9af73ae4b49ebe643d09d85a3aaf1bc Mon Sep 17 00:00:00 2001 From: Jovan Date: Tue, 5 Jul 2022 23:53:26 +0200 Subject: [PATCH 048/126] Added translation for filename --- video_creation/final_video.py | 10 +++++++++- 1 file changed, 9 insertions(+), 1 deletion(-) diff --git a/video_creation/final_video.py b/video_creation/final_video.py index d706361..be42d86 100755 --- a/video_creation/final_video.py +++ b/video_creation/final_video.py @@ -4,6 +4,7 @@ import os import re from os.path import exists from typing import Dict +import translators as ts from moviepy.editor import ( VideoFileClip, @@ -35,7 +36,14 @@ def name_normalize( name = re.sub(r'([0-9]+)\s?\/\s?([0-9]+)', r'\1 of \2', name) name = re.sub(r'(\w+)\s?\/\s?(\w+)', r'\1 or \2', name) name = re.sub(r'\/', r'', name) - return name + + if os.getenv("POSTLANG") != "": + print_substep("Translating filename...") + translated_name = ts.google(name, to_language=os.getenv("POSTLANG")) + return translated_name + + else: + return name def make_final_video(number_of_clips: int, length: int, reddit_obj: dict): From 3e1b95e708aaed434d10b3aa149b2b6f2edd37d1 Mon Sep 17 00:00:00 2001 From: Callum Leslie Date: Tue, 5 Jul 2022 23:22:35 +0100 Subject: [PATCH 049/126] fix: wrongly typed boolean --- reddit/subreddit.py | 2 +- utils/subreddit.py | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/reddit/subreddit.py b/reddit/subreddit.py index 5eed982..40c82a1 100644 --- a/reddit/subreddit.py +++ b/reddit/subreddit.py @@ -17,7 +17,7 @@ def get_subreddit_threads(POST_ID: str): print_substep("Logging into Reddit.") content = {} - if settings.config["reddit"]["creds"]["2fa"] == "true": + if settings.config["reddit"]["creds"]["2fa"] == True: print("\nEnter your two-factor authentication code from your authenticator app.\n") code = input("> ") print() diff --git a/utils/subreddit.py b/utils/subreddit.py index a5d7aa6..140f4d3 100644 --- a/utils/subreddit.py +++ b/utils/subreddit.py @@ -22,7 +22,7 @@ def get_subreddit_undone(submissions: list, subreddit): continue if submission.over_18: try: - if settings.config["settings"]["allow_nsfw"] == "false": + if settings.config["settings"]["allow_nsfw"] == False: print_substep("NSFW Post Detected. Skipping...") continue except AttributeError: From 436266033ec0614addbedc74486179f38de7eade Mon Sep 17 00:00:00 2001 From: Callum Leslie Date: Wed, 6 Jul 2022 00:23:27 +0100 Subject: [PATCH 050/126] refactor: tts engines now use toml config --- TTS/GTTS.py | 4 ++-- TTS/TikTok.py | 4 ++-- TTS/aws_polly.py | 6 +++--- TTS/engine_wrapper.py | 6 +++--- TTS/streamlabs_polly.py | 8 +++----- 5 files changed, 13 insertions(+), 15 deletions(-) diff --git a/TTS/GTTS.py b/TTS/GTTS.py index 992eeb5..27b6934 100644 --- a/TTS/GTTS.py +++ b/TTS/GTTS.py @@ -1,6 +1,6 @@ #!/usr/bin/env python3 import random -import os +from utils import settings from gtts import gTTS max_chars = 0 @@ -12,7 +12,7 @@ class GTTS: self.voices = [] def run(self, text, filepath): - tts = gTTS(text=text, lang=os.getenv("POSTLANG") or "en", slow=False) + tts = gTTS(text=text, lang=settings.config["reddit"]["thread"]["post_lang"] or "en", slow=False) tts.save(filepath) def randomvoice(self): diff --git a/TTS/TikTok.py b/TTS/TikTok.py index 91ba526..4583358 100644 --- a/TTS/TikTok.py +++ b/TTS/TikTok.py @@ -1,5 +1,5 @@ import base64 -import os +from utils import settings import random import requests from requests.adapters import HTTPAdapter, Retry @@ -75,7 +75,7 @@ class TikTok: # TikTok Text-to-Speech Wrapper voice = ( self.randomvoice() if random_voice - else (os.getenv("TIKTOK_VOICE") or random.choice(self.voices["human"])) + else (settings.config["settings"]["tts"]["tiktok_voice"] or random.choice(self.voices["human"])) ) try: r = requests.post(f"{self.URI_BASE}{voice}&req_text={text}&speaker_map_type=0") diff --git a/TTS/aws_polly.py b/TTS/aws_polly.py index 6cbe4c5..94143db 100644 --- a/TTS/aws_polly.py +++ b/TTS/aws_polly.py @@ -2,7 +2,7 @@ from boto3 import Session from botocore.exceptions import BotoCoreError, ClientError import sys -import os +from utils import settings import random voices = [ @@ -35,11 +35,11 @@ class AWSPolly: if random_voice: voice = self.randomvoice() else: - if not os.getenv("AWS_VOICE"): + if not settings.config["settings"]["tts"]["aws_polly_voice"]: return ValueError( f"Please set the environment variable AWS_VOICE to a valid voice. options are: {voices}" ) - voice = str(os.getenv("AWS_VOICE")).capitalize() + voice = str(settings.config["settings"]["tts"]["aws_polly_voice"]).capitalize() try: # Request speech synthesis response = polly.synthesize_speech( diff --git a/TTS/engine_wrapper.py b/TTS/engine_wrapper.py index 13ff850..c520ab0 100644 --- a/TTS/engine_wrapper.py +++ b/TTS/engine_wrapper.py @@ -2,7 +2,6 @@ from pathlib import Path from typing import Tuple import re -from os import getenv # import sox # from mutagen import MutagenError @@ -12,6 +11,7 @@ from rich.progress import track from moviepy.editor import AudioFileClip, CompositeAudioClip, concatenate_audioclips from utils.console import print_step, print_substep from utils.voice import sanitize_text +from utils import settings DEFUALT_MAX_LENGTH: int = 50 # video length variable @@ -56,7 +56,7 @@ class TTSEngine: print_step("Saving Text to MP3 files...") self.call_tts("title", self.reddit_object["thread_title"]) - if self.reddit_object["thread_post"] != "" and getenv("STORYMODE", "").casefold() == "true": + if self.reddit_object["thread_post"] != "" and settings.config["settings"]["storymode"] == True: self.call_tts("posttext", self.reddit_object["thread_post"]) idx = None @@ -109,7 +109,7 @@ class TTSEngine: clip.close() def process_text(text: str): - lang = getenv("POSTLANG", "") + lang = settings.config["reddit"]["thread"]["post_lang"] new_text = sanitize_text(text) if lang: print_substep("Translating Text...") diff --git a/TTS/streamlabs_polly.py b/TTS/streamlabs_polly.py index 066fa53..f880b98 100644 --- a/TTS/streamlabs_polly.py +++ b/TTS/streamlabs_polly.py @@ -1,7 +1,7 @@ import random -import os import requests from requests.exceptions import JSONDecodeError +from utils import settings voices = [ "Brian", @@ -35,11 +35,11 @@ class StreamlabsPolly: if random_voice: voice = self.randomvoice() else: - if not os.getenv("STREAMLABS_VOICE"): + if not settings.config["settings"]["tts"]["streamlabs_polly_voice"]: return ValueError( f"Please set the environment variable STREAMLABS_VOICE to a valid voice. options are: {voices}" ) - voice = str(os.getenv("STREAMLABS_VOICE")).capitalize() + voice = str(settings.config["settings"]["tts"]["streamlabs_polly_voice"]).capitalize() body = {"voice": voice, "text": text, "service": "polly"} response = requests.post(self.url, data=body) try: @@ -56,5 +56,3 @@ class StreamlabsPolly: def randomvoice(self): return random.choice(self.voices) - -# StreamlabsPolly().run(text=str('hi hi ' * 92)[1:], filepath='hello.mp3', random_voice=True) From f5560cae6999e6a44491c85024aca68653d1831a Mon Sep 17 00:00:00 2001 From: Callum Leslie Date: Wed, 6 Jul 2022 00:26:00 +0100 Subject: [PATCH 051/126] refactor: background credit is now a variable --- main.py | 4 ++-- utils/videos.py | 6 ++---- video_creation/background.py | 7 ++++--- video_creation/final_video.py | 12 ++++++------ 4 files changed, 14 insertions(+), 15 deletions(-) diff --git a/main.py b/main.py index 7b7c726..f98e8db 100755 --- a/main.py +++ b/main.py @@ -38,8 +38,8 @@ def main(POST_ID=None): length = math.ceil(length) download_screenshots_of_reddit_posts(reddit_object, number_of_comments) download_background() - chop_background_video(length) - make_final_video(number_of_comments, length, reddit_object) + credit = chop_background_video(length) + make_final_video(number_of_comments, length, reddit_object, credit) def run_many(times): diff --git a/utils/videos.py b/utils/videos.py index 6377256..ec09362 100755 --- a/utils/videos.py +++ b/utils/videos.py @@ -34,7 +34,7 @@ def check_done( return redditobj -def save_data(filename: str, reddit_title: str, reddit_id: str): +def save_data(filename: str, reddit_title: str, reddit_id: str, credit: str): """Saves the videos that have already been generated to a JSON file in video_creation/data/videos.json Args: @@ -50,9 +50,7 @@ def save_data(filename: str, reddit_title: str, reddit_id: str): payload = { "id": reddit_id, "time": str(int(time.time())), - "background_credit": settings.config["settings"]["background_credit"] - if "background_credit" in settings.config["settings"] - else "", + "background_credit": credit, "reddit_title": reddit_title, "filename": filename, } diff --git a/video_creation/background.py b/video_creation/background.py index 2654499..9fd68a1 100644 --- a/video_creation/background.py +++ b/video_creation/background.py @@ -1,5 +1,5 @@ import random -from os import listdir, environ +from os import listdir from pathlib import Path from random import randrange from typing import Tuple @@ -55,7 +55,7 @@ def download_background(): print_substep("Background videos downloaded successfully! 🎉", style="bold green") -def chop_background_video(video_length: int): +def chop_background_video(video_length: int) -> str: """Generates the background footage to be used in the video and writes it to assets/temp/background.mp4 Args: @@ -63,7 +63,7 @@ def chop_background_video(video_length: int): """ print_step("Finding a spot in the backgrounds video to chop...✂️") choice = random.choice(listdir("assets/backgrounds")) - environ["background_credit"] = choice.split("-")[0] + credit = choice.split("-")[0] background = VideoFileClip(f"assets/backgrounds/{choice}") @@ -81,3 +81,4 @@ def chop_background_video(video_length: int): new = video.subclip(start_time, end_time) new.write_videofile("assets/temp/background.mp4") print_substep("Background video chopped successfully!", style="bold green") + return credit diff --git a/video_creation/final_video.py b/video_creation/final_video.py index 84698c5..9b85545 100755 --- a/video_creation/final_video.py +++ b/video_creation/final_video.py @@ -20,13 +20,13 @@ from rich.console import Console from utils.cleanup import cleanup from utils.console import print_step, print_substep from utils.videos import save_data - +from utils import settings console = Console() W, H = 1080, 1920 -def make_final_video(number_of_clips: int, length: int, reddit_obj: dict): +def make_final_video(number_of_clips: int, length: int, reddit_obj: dict, background_credit: str): """Gathers audio clips, gathers all screenshots, stitches them together and saves the final video to assets/temp Args: @@ -37,7 +37,7 @@ def make_final_video(number_of_clips: int, length: int, reddit_obj: dict): print_step("Creating the final video 🎥") VideoFileClip.reW = lambda clip: clip.resize(width=W) VideoFileClip.reH = lambda clip: clip.resize(width=H) - opacity = os.getenv("OPACITY") + opacity = settings.config["settings"]["opacity"] background_clip = ( VideoFileClip("assets/temp/background.mp4") .without_audio() @@ -91,9 +91,9 @@ def make_final_video(number_of_clips: int, length: int, reddit_obj: dict): title = re.sub(r"[^\w\s-]", "", reddit_obj["thread_title"]) idx = re.sub(r"[^\w\s-]", "", reddit_obj["thread_id"]) filename = f"{title}.mp4" - subreddit = os.getenv("SUBREDDIT") + subreddit = settings.config["reddit"]["thread"]["subreddit"] - save_data(filename, title, idx) + save_data(filename, title, idx, credit) if not exists(f"./results/{subreddit}"): print_substep("The results folder didn't exist so I made it") @@ -118,5 +118,5 @@ def make_final_video(number_of_clips: int, length: int, reddit_obj: dict): print_substep("See result in the results folder!") print_step( - f'Reddit title: {reddit_obj["thread_title"]} \n Background Credit: {os.getenv("background_credit")}' + f'Reddit title: {reddit_obj["thread_title"]} \n Background Credit: {background_credit}' ) From 9c87cb7d950f913be5a369a1690d4456d75c942b Mon Sep 17 00:00:00 2001 From: Callum Leslie Date: Wed, 6 Jul 2022 00:28:51 +0100 Subject: [PATCH 052/126] fix: mismatched variable name --- video_creation/final_video.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/video_creation/final_video.py b/video_creation/final_video.py index 9b85545..fb54e96 100755 --- a/video_creation/final_video.py +++ b/video_creation/final_video.py @@ -93,7 +93,7 @@ def make_final_video(number_of_clips: int, length: int, reddit_obj: dict, backgr filename = f"{title}.mp4" subreddit = settings.config["reddit"]["thread"]["subreddit"] - save_data(filename, title, idx, credit) + save_data(filename, title, idx, background_credit) if not exists(f"./results/{subreddit}"): print_substep("The results folder didn't exist so I made it") From ead88d93a1216b1023b00b2de92889acc10bb04a Mon Sep 17 00:00:00 2001 From: Callum Leslie Date: Wed, 6 Jul 2022 00:27:26 +0100 Subject: [PATCH 053/126] refactor: screenshot downloader now uses toml config --- video_creation/screenshot_downloader.py | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/video_creation/screenshot_downloader.py b/video_creation/screenshot_downloader.py index c6ed035..523966c 100644 --- a/video_creation/screenshot_downloader.py +++ b/video_creation/screenshot_downloader.py @@ -1,5 +1,4 @@ import json -import os from pathlib import Path from typing import Dict @@ -58,7 +57,7 @@ def download_screenshots_of_reddit_posts(reddit_object: dict, screenshot_num: in if settings.config["reddit"]["thread"]["post_lang"]: print_substep("Translating post...") - texts_in_tl = ts.google(reddit_object["thread_title"], to_language=os.getenv("POSTLANG")) + texts_in_tl = ts.google(reddit_object["thread_title"], to_language=settings.config["reddit"]["thread"]["post_lang"]) page.evaluate( "tl_content => document.querySelector('[data-test-id=\"post-content\"] > div:nth-child(3) > div > div').textContent = tl_content", From ac28e72017da0fddd23b8a9bb4a02f4593e9b637 Mon Sep 17 00:00:00 2001 From: Callum Leslie Date: Wed, 6 Jul 2022 00:40:46 +0100 Subject: [PATCH 054/126] fix: do not use optional so as to avoid undefined values --- .config.template.toml | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/.config.template.toml b/.config.template.toml index 7f4a277..9fbf3f3 100644 --- a/.config.template.toml +++ b/.config.template.toml @@ -3,21 +3,21 @@ client_id = { optional = false, nmin = 12, nmax = 30, explanation = "the ID of y client_secret = { optional = false, nmin = 20, nmax = 40, explanation = "the SECRET of your Reddit app of SCRIPT type", example = "fFAGRNJru1FTz70BzhT3Zg", regex = "^[-a-zA-Z0-9._~+/]+=*$", input_error = "The client ID can only contain printable characters.", oob_error = "The secret should be over 20 and under 40 characters, double check your input." } username = { optional = false, nmin = 3, nmax = 20, explanation = "the username of your reddit account", example = "asdfghjkl", regex = "^[-_0-9a-zA-Z]+$", oob_error = "A username HAS to be between 3 and 20 characters" } password = { optional = false, nmin = 8, explanation = "the password of your reddit account", example = "fFAGRNJru1FTz70BzhT3Zg", oob_error = "Password too short" } -2fa = { optional = true, type = "bool", options = [ +2fa = { optional = false, type = "bool", options = [ true, false, ], default = false, explanation = "Whether you have Reddit 2FA enabled, Valid options are True and False", example = true } [reddit.thread] -random = { optional = true, options = [ +random = { optional = false, options = [ true, false, ], default = false, type = "bool", explanation = "If set to no, it will ask you a thread link to extract the thread, if yes it will randomize it. Default: 'False'", example = "True" } subreddit = { optional = false, regex = "[_0-9a-zA-Z]+$", nmin = 3, nmax = 21, explanation = "what subreddit to pull posts from, the name of the sub, not the URL", example = "AskReddit", oob_error = "A subreddit name HAS to be between 3 and 20 characters" } -post_id = { optional = true, regex = "^((?!://|://).)*$", explanation = "Used if you want to use a specific post.", example = "urdtfx" } +post_id = { optional = false, default = "", regex = "^((?!://|://).)*$", explanation = "Used if you want to use a specific post.", example = "urdtfx" } max_comment_length = { default = 500, optional = false, nmin = 10, nmax = 10000, type = "int", explanation = "max number of characters a comment can have. default is 500", example = 500, oob_error = "the max comment length should be between 10 and 10000" } -post_lang = { optional = true, explanation = "The language you would like to translate to - leave blank for none.", example = "es-cr"} +post_lang = { default = "", optional = false, explanation = "The language you would like to translate to - leave blank for none.", example = "es-cr"} [settings] allow_nsfw = { optional = false, type = "bool", default = false, example = false, options = [ @@ -30,7 +30,7 @@ theme = { optional = false, default = "light", example = "dark", options = [ ], explanation = "sets the Reddit theme, either LIGHT or DARK" } times_to_run = { optional = false, default = 1, example = 2, explanation = "used if you want to run multiple times. set to an int e.g. 4 or 29 or 1", type = "int", nmin = 1, oob_error = "It's very hard to run something less than once." } opacity = { optional = false, default = 0.9, example = 0.8, explanation = "Sets the opacity of the comments when overlayed over the background", type = "float", nmin = 0, nmax = 1, oob_error = "The opacity HAS to be between 0 and 1", input_error = "The opacity HAS to be a decimal number between 0 and 1" } -storymode = { optional = true, type = "bool", default = false, example = false, options = [ +storymode = { optional = false, type = "bool", default = false, example = false, options = [ true, false, ] } @@ -40,4 +40,4 @@ storymode = { optional = true, type = "bool", default = false, example = false, aws_polly_voice = { optional = false, default = "Matthew", example = "Matthew", explanation = "The voice used for AWS Polly" } streamlabs_polly_voice = { optional = false, default = "Matthew", example = "Matthew", explanation = "The voice used for Streamlabs Polly" } tiktok_voice = { optional = false, default = "en_us_006", example = "en_us_006", explanation = "The voice used for TikTok TTS" } -choice = { optional = true, example = "streamlabspolly", explanation = "The backend used for TTS generation" } +choice = { optional = false, default = "", options = [ "streamlabspolly", "tiktok", "googletranslate", "awspolly", ""], example = "streamlabspolly", explanation = "The backend used for TTS generation. This can be left blank and you will be prompted to choose at runtime." } From 6a16c9f3e7a295e178857ccf5feab3e207de6394 Mon Sep 17 00:00:00 2001 From: Callum Leslie Date: Wed, 6 Jul 2022 00:46:52 +0100 Subject: [PATCH 055/126] style: format with python-black --- TTS/GTTS.py | 6 ++++- TTS/TikTok.py | 17 +++++++++----- TTS/aws_polly.py | 4 +++- TTS/engine_wrapper.py | 22 +++++++++++++----- TTS/streamlabs_polly.py | 5 ++-- main.py | 4 +++- reddit/subreddit.py | 31 ++++++++++++++++++------- utils/cleanup.py | 4 +++- utils/console.py | 9 +++++-- utils/settings.py | 28 ++++++++++++++++++---- utils/subreddit.py | 4 +++- utils/videos.py | 4 +++- video_creation/background.py | 4 +++- video_creation/final_video.py | 22 +++++++++++++----- video_creation/screenshot_downloader.py | 20 ++++++++++++---- video_creation/voices.py | 14 ++++++++--- 16 files changed, 148 insertions(+), 50 deletions(-) diff --git a/TTS/GTTS.py b/TTS/GTTS.py index 27b6934..31e29df 100644 --- a/TTS/GTTS.py +++ b/TTS/GTTS.py @@ -12,7 +12,11 @@ class GTTS: self.voices = [] def run(self, text, filepath): - tts = gTTS(text=text, lang=settings.config["reddit"]["thread"]["post_lang"] or "en", slow=False) + tts = gTTS( + text=text, + lang=settings.config["reddit"]["thread"]["post_lang"] or "en", + slow=False, + ) tts.save(filepath) def randomvoice(self): diff --git a/TTS/TikTok.py b/TTS/TikTok.py index 4583358..6a116d7 100644 --- a/TTS/TikTok.py +++ b/TTS/TikTok.py @@ -62,9 +62,7 @@ noneng = [ class TikTok: # TikTok Text-to-Speech Wrapper def __init__(self): - self.URI_BASE = ( - "https://api16-normal-useast5.us.tiktokv.com/media/api/text/speech/invoke/?text_speaker=" - ) + self.URI_BASE = "https://api16-normal-useast5.us.tiktokv.com/media/api/text/speech/invoke/?text_speaker=" self.max_chars = 300 self.voices = {"human": human, "nonhuman": nonhuman, "noneng": noneng} @@ -75,10 +73,15 @@ class TikTok: # TikTok Text-to-Speech Wrapper voice = ( self.randomvoice() if random_voice - else (settings.config["settings"]["tts"]["tiktok_voice"] or random.choice(self.voices["human"])) + else ( + settings.config["settings"]["tts"]["tiktok_voice"] + or random.choice(self.voices["human"]) + ) ) try: - r = requests.post(f"{self.URI_BASE}{voice}&req_text={text}&speaker_map_type=0") + r = requests.post( + f"{self.URI_BASE}{voice}&req_text={text}&speaker_map_type=0" + ) except requests.exceptions.SSLError: # https://stackoverflow.com/a/47475019/18516611 session = requests.Session() @@ -86,7 +89,9 @@ class TikTok: # TikTok Text-to-Speech Wrapper adapter = HTTPAdapter(max_retries=retry) session.mount("http://", adapter) session.mount("https://", adapter) - r = session.post(f"{self.URI_BASE}{voice}&req_text={text}&speaker_map_type=0") + r = session.post( + f"{self.URI_BASE}{voice}&req_text={text}&speaker_map_type=0" + ) # print(r.text) vstr = [r.json()["data"]["v_str"]][0] b64d = base64.b64decode(vstr) diff --git a/TTS/aws_polly.py b/TTS/aws_polly.py index 94143db..bf8ec1e 100644 --- a/TTS/aws_polly.py +++ b/TTS/aws_polly.py @@ -39,7 +39,9 @@ class AWSPolly: return ValueError( f"Please set the environment variable AWS_VOICE to a valid voice. options are: {voices}" ) - voice = str(settings.config["settings"]["tts"]["aws_polly_voice"]).capitalize() + voice = str( + settings.config["settings"]["tts"]["aws_polly_voice"] + ).capitalize() try: # Request speech synthesis response = polly.synthesize_speech( diff --git a/TTS/engine_wrapper.py b/TTS/engine_wrapper.py index c520ab0..e3cc04b 100644 --- a/TTS/engine_wrapper.py +++ b/TTS/engine_wrapper.py @@ -56,11 +56,16 @@ class TTSEngine: print_step("Saving Text to MP3 files...") self.call_tts("title", self.reddit_object["thread_title"]) - if self.reddit_object["thread_post"] != "" and settings.config["settings"]["storymode"] == True: + if ( + self.reddit_object["thread_post"] != "" + and settings.config["settings"]["storymode"] == True + ): self.call_tts("posttext", self.reddit_object["thread_post"]) idx = None - for idx, comment in track(enumerate(self.reddit_object["comments"]), "Saving..."): + for idx, comment in track( + enumerate(self.reddit_object["comments"]), "Saving..." + ): # ! Stop creating mp3 files if the length is greater than max length. if self.length > self.max_length: break @@ -76,7 +81,9 @@ class TTSEngine: split_files = [] split_text = [ x.group().strip() - for x in re.finditer(rf" *((.{{0,{self.tts_module.max_chars}}})(\.|.$))", text) + for x in re.finditer( + rf" *((.{{0,{self.tts_module.max_chars}}})(\.|.$))", text + ) ] idy = None @@ -94,12 +101,14 @@ class TTSEngine: Path(name).unlink() # for i in range(0, idy + 1): - # print(f"Cleaning up {self.path}/{idx}-{i}.part.mp3") + # print(f"Cleaning up {self.path}/{idx}-{i}.part.mp3") - # Path(f"{self.path}/{idx}-{i}.part.mp3").unlink() + # Path(f"{self.path}/{idx}-{i}.part.mp3").unlink() def call_tts(self, filename: str, text: str): - self.tts_module.run(text=process_text(text), filepath=f"{self.path}/{filename}.mp3") + self.tts_module.run( + text=process_text(text), filepath=f"{self.path}/{filename}.mp3" + ) # try: # self.length += MP3(f"{self.path}/{filename}.mp3").info.length # except (MutagenError, HeaderNotFoundError): @@ -108,6 +117,7 @@ class TTSEngine: self.length += clip.duration clip.close() + def process_text(text: str): lang = settings.config["reddit"]["thread"]["post_lang"] new_text = sanitize_text(text) diff --git a/TTS/streamlabs_polly.py b/TTS/streamlabs_polly.py index f880b98..e9e6358 100644 --- a/TTS/streamlabs_polly.py +++ b/TTS/streamlabs_polly.py @@ -39,7 +39,9 @@ class StreamlabsPolly: return ValueError( f"Please set the environment variable STREAMLABS_VOICE to a valid voice. options are: {voices}" ) - voice = str(settings.config["settings"]["tts"]["streamlabs_polly_voice"]).capitalize() + voice = str( + settings.config["settings"]["tts"]["streamlabs_polly_voice"] + ).capitalize() body = {"voice": voice, "text": text, "service": "polly"} response = requests.post(self.url, data=body) try: @@ -55,4 +57,3 @@ class StreamlabsPolly: def randomvoice(self): return random.choice(self.voices) - diff --git a/main.py b/main.py index f98e8db..c7079d5 100755 --- a/main.py +++ b/main.py @@ -59,7 +59,9 @@ if __name__ == "__main__": run_many(config["settings"]["times_to_run"]) elif len(config["reddit"]["thread"]["post_id"].split("+")) > 1: - for index, post_id in enumerate(config["reddit"]["thread"]["post_id"].split("+")): + for index, post_id in enumerate( + config["reddit"]["thread"]["post_id"].split("+") + ): index += 1 print_step( f'on the {index}{("st" if index%10 == 1 else ("nd" if index%10 == 2 else ("rd" if index%10 == 3 else "th")))} post of {len(config["reddit"]["thread"]["post_id"].split("+"))}' diff --git a/reddit/subreddit.py b/reddit/subreddit.py index 40c82a1..7583653 100644 --- a/reddit/subreddit.py +++ b/reddit/subreddit.py @@ -18,7 +18,9 @@ def get_subreddit_threads(POST_ID: str): content = {} if settings.config["reddit"]["creds"]["2fa"] == True: - print("\nEnter your two-factor authentication code from your authenticator app.\n") + print( + "\nEnter your two-factor authentication code from your authenticator app.\n" + ) code = input("> ") print() pw = settings.config["reddit"]["creds"]["password"] @@ -29,7 +31,7 @@ def get_subreddit_threads(POST_ID: str): if username.casefold().startswith("u/"): username = username[2:] reddit = praw.Reddit( - client_id= settings.config["reddit"]["creds"]["client_id"], + client_id=settings.config["reddit"]["creds"]["client_id"], client_secret=settings.config["reddit"]["creds"]["client_secret"], user_agent="Accessing Reddit threads", username=username, @@ -39,10 +41,14 @@ def get_subreddit_threads(POST_ID: str): # Ask user for subreddit input print_step("Getting subreddit threads...") - if not settings.config["reddit"]["thread"]["subreddit"]: # note to user. you can have multiple subreddits via reddit.subreddit("redditdev+learnpython") + if not settings.config["reddit"]["thread"][ + "subreddit" + ]: # note to user. you can have multiple subreddits via reddit.subreddit("redditdev+learnpython") try: subreddit = reddit.subreddit( - re.sub(r"r\/", "", input("What subreddit would you like to pull from? ")) + re.sub( + r"r\/", "", input("What subreddit would you like to pull from? ") + ) # removes the r/ from the input ) except ValueError: @@ -52,7 +58,9 @@ def get_subreddit_threads(POST_ID: str): sub = settings.config["reddit"]["thread"]["subreddit"] print_substep(f"Using subreddit: r/{sub} from TOML config") subreddit_choice = sub - if subreddit_choice.casefold().startswith("r/"): # removes the r/ from the input + if subreddit_choice.casefold().startswith( + "r/" + ): # removes the r/ from the input subreddit_choice = subreddit_choice[2:] subreddit = reddit.subreddit( subreddit_choice @@ -60,8 +68,13 @@ def get_subreddit_threads(POST_ID: str): if POST_ID: # would only be called if there are multiple queued posts submission = reddit.submission(id=POST_ID) - elif settings.config["reddit"]["thread"]["post_id"] and len(settings.config["reddit"]["thread"]["post_id"].split("+")) == 1: - submission = reddit.submission(id=settings.config["reddit"]["thread"]["post_id"]) + elif ( + settings.config["reddit"]["thread"]["post_id"] + and len(settings.config["reddit"]["thread"]["post_id"].split("+")) == 1 + ): + submission = reddit.submission( + id=settings.config["reddit"]["thread"]["post_id"] + ) else: threads = subreddit.hot(limit=25) @@ -90,7 +103,9 @@ def get_subreddit_threads(POST_ID: str): if top_level_comment.body in ["[removed]", "[deleted]"]: continue # # see https://github.com/JasonLovesDoggo/RedditVideoMakerBot/issues/78 if not top_level_comment.stickied: - if len(top_level_comment.body) <= int(settings.config["reddit"]["thread"]["max_comment_length"]): + if len(top_level_comment.body) <= int( + settings.config["reddit"]["thread"]["max_comment_length"] + ): if ( top_level_comment.author is not None ): # if errors occur with this change to if not. diff --git a/utils/cleanup.py b/utils/cleanup.py index ef4fc44..44629a9 100644 --- a/utils/cleanup.py +++ b/utils/cleanup.py @@ -10,7 +10,9 @@ def cleanup() -> int: """ if exists("./assets/temp"): count = 0 - files = [f for f in os.listdir(".") if f.endswith(".mp4") and "temp" in f.lower()] + files = [ + f for f in os.listdir(".") if f.endswith(".mp4") and "temp" in f.lower() + ] count += len(files) for f in files: os.remove(f) diff --git a/utils/console.py b/utils/console.py index 310247d..b8faa80 100644 --- a/utils/console.py +++ b/utils/console.py @@ -49,7 +49,10 @@ def handle_input( optional=False, ): if optional: - console.print(message + "\n[green]This is an optional value. Do you want to skip it? (y/n)") + console.print( + message + + "\n[green]This is an optional value. Do you want to skip it? (y/n)" + ) if input().casefold().startswith("y"): return None if default is not NotImplemented: @@ -84,7 +87,9 @@ def handle_input( continue elif match != "" and re.match(match, user_input) is None: console.print( - "[red]" + err_message + "\nAre you absolutely sure it's correct?(y/n)" + "[red]" + + err_message + + "\nAre you absolutely sure it's correct?(y/n)" ) if input().casefold().startswith("y"): break diff --git a/utils/settings.py b/utils/settings.py index d674974..c77e4ae 100755 --- a/utils/settings.py +++ b/utils/settings.py @@ -54,7 +54,11 @@ def check(value, checks, name): and not hasattr(value, "__iter__") and ( ("nmin" in checks and checks["nmin"] is not None and value < checks["nmin"]) - or ("nmax" in checks and checks["nmax"] is not None and value > checks["nmax"]) + or ( + "nmax" in checks + and checks["nmax"] is not None + and value > checks["nmax"] + ) ) ): incorrect = True @@ -62,8 +66,16 @@ def check(value, checks, name): not incorrect and hasattr(value, "__iter__") and ( - ("nmin" in checks and checks["nmin"] is not None and len(value) < checks["nmin"]) - or ("nmax" in checks and checks["nmax"] is not None and len(value) > checks["nmax"]) + ( + "nmin" in checks + and checks["nmin"] is not None + and len(value) < checks["nmin"] + ) + or ( + "nmax" in checks + and checks["nmax"] is not None + and len(value) > checks["nmax"] + ) ) ): incorrect = True @@ -71,7 +83,11 @@ def check(value, checks, name): if incorrect: value = handle_input( message=( - (("[blue]Example: " + str(checks["example"]) + "\n") if "example" in checks else "") + ( + ("[blue]Example: " + str(checks["example"]) + "\n") + if "example" in checks + else "" + ) + "[red]" + ("Non-optional ", "Optional ")[ "optional" in checks and checks["optional"] is True @@ -84,7 +100,9 @@ def check(value, checks, name): check_type=eval(checks["type"]) if "type" in checks else False, default=checks["default"] if "default" in checks else NotImplemented, match=checks["regex"] if "regex" in checks else "", - err_message=checks["input_error"] if "input_error" in checks else "Incorrect input", + err_message=checks["input_error"] + if "input_error" in checks + else "Incorrect input", nmin=checks["nmin"] if "nmin" in checks else None, nmax=checks["nmax"] if "nmax" in checks else None, oob_error=checks["oob_error"] diff --git a/utils/subreddit.py b/utils/subreddit.py index 140f4d3..7647454 100644 --- a/utils/subreddit.py +++ b/utils/subreddit.py @@ -15,7 +15,9 @@ def get_subreddit_undone(submissions: list, subreddit): """ # recursively checks if the top submission in the list was already done. - with open("./video_creation/data/videos.json", "r", encoding="utf-8") as done_vids_raw: + with open( + "./video_creation/data/videos.json", "r", encoding="utf-8" + ) as done_vids_raw: done_videos = json.load(done_vids_raw) for submission in submissions: if already_done(done_videos, submission): diff --git a/utils/videos.py b/utils/videos.py index ec09362..38184e4 100755 --- a/utils/videos.py +++ b/utils/videos.py @@ -20,7 +20,9 @@ def check_done( Returns: Dict[str]|None: Reddit object in args """ - with open("./video_creation/data/videos.json", "r", encoding="utf-8") as done_vids_raw: + with open( + "./video_creation/data/videos.json", "r", encoding="utf-8" + ) as done_vids_raw: done_videos = json.load(done_vids_raw) for video in done_videos: if video["id"] == str(redditobj): diff --git a/video_creation/background.py b/video_creation/background.py index 9fd68a1..494c7d2 100644 --- a/video_creation/background.py +++ b/video_creation/background.py @@ -52,7 +52,9 @@ def download_background(): "assets/backgrounds", filename=f"{credit}-{filename}" ) - print_substep("Background videos downloaded successfully! 🎉", style="bold green") + print_substep( + "Background videos downloaded successfully! 🎉", style="bold green" + ) def chop_background_video(video_length: int) -> str: diff --git a/video_creation/final_video.py b/video_creation/final_video.py index fb54e96..8fccd38 100755 --- a/video_creation/final_video.py +++ b/video_creation/final_video.py @@ -21,12 +21,15 @@ from utils.cleanup import cleanup from utils.console import print_step, print_substep from utils.videos import save_data from utils import settings + console = Console() W, H = 1080, 1920 -def make_final_video(number_of_clips: int, length: int, reddit_obj: dict, background_credit: str): +def make_final_video( + number_of_clips: int, length: int, reddit_obj: dict, background_credit: str +): """Gathers audio clips, gathers all screenshots, stitches them together and saves the final video to assets/temp Args: @@ -46,7 +49,9 @@ def make_final_video(number_of_clips: int, length: int, reddit_obj: dict, backgr ) # Gather all audio clips - audio_clips = [AudioFileClip(f"assets/temp/mp3/{i}.mp3") for i in range(number_of_clips)] + audio_clips = [ + AudioFileClip(f"assets/temp/mp3/{i}.mp3") for i in range(number_of_clips) + ] audio_clips.insert(0, AudioFileClip("assets/temp/mp3/title.mp3")) audio_concat = concatenate_audioclips(audio_clips) audio_composite = CompositeAudioClip([audio_concat]) @@ -63,7 +68,7 @@ def make_final_video(number_of_clips: int, length: int, reddit_obj: dict, backgr .set_duration(audio_clips[0].duration) .set_position("center") .resize(width=W - 100) - .set_opacity(new_opacity) + .set_opacity(new_opacity), ) for i in range(0, number_of_clips): @@ -85,13 +90,15 @@ def make_final_video(number_of_clips: int, length: int, reddit_obj: dict, backgr # .set_opacity(float(opacity)), # ) # else: - image_concat = concatenate_videoclips(image_clips).set_position(("center", "center")) + image_concat = concatenate_videoclips(image_clips).set_position( + ("center", "center") + ) image_concat.audio = audio_composite final = CompositeVideoClip([background_clip, image_concat]) title = re.sub(r"[^\w\s-]", "", reddit_obj["thread_title"]) idx = re.sub(r"[^\w\s-]", "", reddit_obj["thread_id"]) filename = f"{title}.mp4" - subreddit = settings.config["reddit"]["thread"]["subreddit"] + subreddit = settings.config["reddit"]["thread"]["subreddit"] save_data(filename, title, idx, background_credit) @@ -108,7 +115,10 @@ def make_final_video(number_of_clips: int, length: int, reddit_obj: dict, backgr threads=multiprocessing.cpu_count(), ) ffmpeg_tools.ffmpeg_extract_subclip( - "assets/temp/temp.mp4", 0, final.duration, targetname=f"results/{subreddit}/{filename}" + "assets/temp/temp.mp4", + 0, + final.duration, + targetname=f"results/{subreddit}/{filename}", ) # os.remove("assets/temp/temp.mp4") diff --git a/video_creation/screenshot_downloader.py b/video_creation/screenshot_downloader.py index 523966c..efc48bd 100644 --- a/video_creation/screenshot_downloader.py +++ b/video_creation/screenshot_downloader.py @@ -35,9 +35,13 @@ def download_screenshots_of_reddit_posts(reddit_object: dict, screenshot_num: in context = browser.new_context() if settings.config["settings"]["theme"] == "dark": - cookie_file = open("./video_creation/data/cookie-dark-mode.json", encoding="utf-8") + cookie_file = open( + "./video_creation/data/cookie-dark-mode.json", encoding="utf-8" + ) else: - cookie_file = open("./video_creation/data/cookie-light-mode.json", encoding="utf-8") + cookie_file = open( + "./video_creation/data/cookie-light-mode.json", encoding="utf-8" + ) cookies = json.load(cookie_file) context.add_cookies(cookies) # load preference cookies # Get the thread screenshot @@ -57,7 +61,10 @@ def download_screenshots_of_reddit_posts(reddit_object: dict, screenshot_num: in if settings.config["reddit"]["thread"]["post_lang"]: print_substep("Translating post...") - texts_in_tl = ts.google(reddit_object["thread_title"], to_language=settings.config["reddit"]["thread"]["post_lang"]) + texts_in_tl = ts.google( + reddit_object["thread_title"], + to_language=settings.config["reddit"]["thread"]["post_lang"], + ) page.evaluate( "tl_content => document.querySelector('[data-test-id=\"post-content\"] > div:nth-child(3) > div > div').textContent = tl_content", @@ -66,7 +73,9 @@ def download_screenshots_of_reddit_posts(reddit_object: dict, screenshot_num: in else: print_substep("Skipping translation...") - page.locator('[data-test-id="post-content"]').screenshot(path="assets/temp/png/title.png") + page.locator('[data-test-id="post-content"]').screenshot( + path="assets/temp/png/title.png" + ) if storymode: page.locator('[data-click-id="text"]').screenshot( @@ -89,7 +98,8 @@ def download_screenshots_of_reddit_posts(reddit_object: dict, screenshot_num: in if settings.config["reddit"]["thread"]["post_lang"]: comment_tl = ts.google( - comment["comment_body"], to_language=settings.config["reddit"]["thread"]["post_lang"] + comment["comment_body"], + to_language=settings.config["reddit"]["thread"]["post_lang"], ) page.evaluate( '([tl_content, tl_id]) => document.querySelector(`#t1_${tl_id} > div:nth-child(2) > div > div[data-testid="comment"] > div`).textContent = tl_content', diff --git a/video_creation/voices.py b/video_creation/voices.py index 4224d15..4bbd5d7 100644 --- a/video_creation/voices.py +++ b/video_creation/voices.py @@ -35,7 +35,9 @@ def save_text_to_mp3(reddit_obj) -> Tuple[int, int]: voice = settings.config["settings"]["tts"]["choice"] if voice.casefold() in map(lambda _: _.casefold(), TTSProviders): - text_to_mp3 = TTSEngine(get_case_insensitive_key_value(TTSProviders, voice), reddit_obj) + text_to_mp3 = TTSEngine( + get_case_insensitive_key_value(TTSProviders, voice), reddit_obj + ) else: while True: print_step("Please choose one of the following TTS providers: ") @@ -44,13 +46,19 @@ def save_text_to_mp3(reddit_obj) -> Tuple[int, int]: if choice.casefold() in map(lambda _: _.casefold(), TTSProviders): break print("Unknown Choice") - text_to_mp3 = TTSEngine(get_case_insensitive_key_value(TTSProviders, choice), reddit_obj) + text_to_mp3 = TTSEngine( + get_case_insensitive_key_value(TTSProviders, choice), reddit_obj + ) return text_to_mp3.run() def get_case_insensitive_key_value(input_dict, key): return next( - (value for dict_key, value in input_dict.items() if dict_key.lower() == key.lower()), + ( + value + for dict_key, value in input_dict.items() + if dict_key.lower() == key.lower() + ), None, ) From f5bb3b6129d5e9d3b322c253b5e6fbea9c8560b4 Mon Sep 17 00:00:00 2001 From: Callum Leslie Date: Wed, 6 Jul 2022 00:58:45 +0100 Subject: [PATCH 056/126] chore: update requirements --- requirements.txt | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/requirements.txt b/requirements.txt index 687f952..7ef2ad8 100644 --- a/requirements.txt +++ b/requirements.txt @@ -2,11 +2,10 @@ boto3==1.24.12 botocore==1.27.22 gTTS==2.2.4 moviepy==1.0.3 -mutagen==1.45.1 playwright==1.23.0 praw==7.6.0 -python-dotenv==0.20.0 pytube==12.1.0 requests==2.28.1 rich==12.4.4 +toml==0.10.2 translators==5.3.1 From 53c98f5d512a7beb415632cd4dbca465d309b1ef Mon Sep 17 00:00:00 2001 From: Jason <66544866+JasonLovesDoggo@users.noreply.github.com> Date: Tue, 5 Jul 2022 20:31:21 -0400 Subject: [PATCH 057/126] Revert "Added translation for filename" --- video_creation/final_video.py | 10 +--------- 1 file changed, 1 insertion(+), 9 deletions(-) diff --git a/video_creation/final_video.py b/video_creation/final_video.py index be42d86..d706361 100755 --- a/video_creation/final_video.py +++ b/video_creation/final_video.py @@ -4,7 +4,6 @@ import os import re from os.path import exists from typing import Dict -import translators as ts from moviepy.editor import ( VideoFileClip, @@ -36,14 +35,7 @@ def name_normalize( name = re.sub(r'([0-9]+)\s?\/\s?([0-9]+)', r'\1 of \2', name) name = re.sub(r'(\w+)\s?\/\s?(\w+)', r'\1 or \2', name) name = re.sub(r'\/', r'', name) - - if os.getenv("POSTLANG") != "": - print_substep("Translating filename...") - translated_name = ts.google(name, to_language=os.getenv("POSTLANG")) - return translated_name - - else: - return name + return name def make_final_video(number_of_clips: int, length: int, reddit_obj: dict): From 869bfd874a46ad53439581c387842710031af6f7 Mon Sep 17 00:00:00 2001 From: Callum Leslie Date: Wed, 6 Jul 2022 11:47:44 +0000 Subject: [PATCH 058/126] docs: clarify config in readme --- README.md | 8 +++++--- 1 file changed, 5 insertions(+), 3 deletions(-) diff --git a/README.md b/README.md index dc6237c..7f10287 100644 --- a/README.md +++ b/README.md @@ -40,11 +40,13 @@ The only original thing being done is the editing and gathering of all materials 1. Clone this repository 2. Run `pip install -r requirements.txt` -3. Run `playwright install` and `playwright install-deps`. (if this fails try adding python -m to the front of the command) +3. Run `python -m playwright install` and `python -m playwright install-deps` 4. Run `python main.py` - required\*\*), visit [the Reddit Apps page.](https://www.reddit.com/prefs/apps) TL;DR set up an app that is a "script". -5. Enjoy 😎 +5. Visit [the Reddit Apps page.](https://www.reddit.com/prefs/apps), and set up an app that is a "script". +6. The bot will ask you to fill in your details to connect to the Reddit API, and configure the bot to your liking +7. Enjoy 😎 +8. If you need to reconfigure the bot, simply open the `config.toml` file and delete the lines that need to be changed. On the next run of the bot, it will help you reconfigure those options. (Note if you got an error installing or running the bot try first rerunning the command with a three after the name e.g. python3 or pip3) From 96b57ff4119109ba14354ae5af9893b3cfa76fe9 Mon Sep 17 00:00:00 2001 From: Callum Leslie Date: Wed, 6 Jul 2022 13:14:07 +0100 Subject: [PATCH 059/126] chore: add python-black action --- .github/workflows/lint.yml | 10 ++++++++++ 1 file changed, 10 insertions(+) create mode 100644 .github/workflows/lint.yml diff --git a/.github/workflows/lint.yml b/.github/workflows/lint.yml new file mode 100644 index 0000000..b04fb15 --- /dev/null +++ b/.github/workflows/lint.yml @@ -0,0 +1,10 @@ +name: Lint + +on: [push, pull_request] + +jobs: + lint: + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v2 + - uses: psf/black@stable From 5dbb35128df02225a2d2a423f86deb2dbaef44cc Mon Sep 17 00:00:00 2001 From: Callum Leslie Date: Wed, 6 Jul 2022 13:14:26 +0100 Subject: [PATCH 060/126] chore: update codeql-analysis to run on develop --- .github/workflows/codeql-analysis.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/workflows/codeql-analysis.yml b/.github/workflows/codeql-analysis.yml index 238dad4..ec78b1a 100644 --- a/.github/workflows/codeql-analysis.yml +++ b/.github/workflows/codeql-analysis.yml @@ -14,10 +14,10 @@ name: "CodeQL" on: push: - branches: [ "master" ] + branches: [ "master", "develop" ] pull_request: # The branches below must be a subset of the branches above - branches: [ "master" ] + branches: [ "master", "develop" ] schedule: - cron: '16 14 * * 3' From d0a76972567bfcaebb3f0f069bf29b5e3c06e5da Mon Sep 17 00:00:00 2001 From: Callum Leslie Date: Wed, 6 Jul 2022 14:40:56 +0100 Subject: [PATCH 061/126] fix: optionals return default or empty string --- .config.template.toml | 10 +++++----- utils/console.py | 2 +- utils/settings.py | 32 ++++++++++++-------------------- 3 files changed, 18 insertions(+), 26 deletions(-) diff --git a/.config.template.toml b/.config.template.toml index 9fbf3f3..e1089e3 100644 --- a/.config.template.toml +++ b/.config.template.toml @@ -3,21 +3,21 @@ client_id = { optional = false, nmin = 12, nmax = 30, explanation = "the ID of y client_secret = { optional = false, nmin = 20, nmax = 40, explanation = "the SECRET of your Reddit app of SCRIPT type", example = "fFAGRNJru1FTz70BzhT3Zg", regex = "^[-a-zA-Z0-9._~+/]+=*$", input_error = "The client ID can only contain printable characters.", oob_error = "The secret should be over 20 and under 40 characters, double check your input." } username = { optional = false, nmin = 3, nmax = 20, explanation = "the username of your reddit account", example = "asdfghjkl", regex = "^[-_0-9a-zA-Z]+$", oob_error = "A username HAS to be between 3 and 20 characters" } password = { optional = false, nmin = 8, explanation = "the password of your reddit account", example = "fFAGRNJru1FTz70BzhT3Zg", oob_error = "Password too short" } -2fa = { optional = false, type = "bool", options = [ +2fa = { optional = true, type = "bool", options = [ true, false, ], default = false, explanation = "Whether you have Reddit 2FA enabled, Valid options are True and False", example = true } [reddit.thread] -random = { optional = false, options = [ +random = { optional = true, options = [ true, false, ], default = false, type = "bool", explanation = "If set to no, it will ask you a thread link to extract the thread, if yes it will randomize it. Default: 'False'", example = "True" } subreddit = { optional = false, regex = "[_0-9a-zA-Z]+$", nmin = 3, nmax = 21, explanation = "what subreddit to pull posts from, the name of the sub, not the URL", example = "AskReddit", oob_error = "A subreddit name HAS to be between 3 and 20 characters" } -post_id = { optional = false, default = "", regex = "^((?!://|://).)*$", explanation = "Used if you want to use a specific post.", example = "urdtfx" } +post_id = { optional = true, default = "", regex = "^((?!://|://).)*$", explanation = "Used if you want to use a specific post.", example = "urdtfx" } max_comment_length = { default = 500, optional = false, nmin = 10, nmax = 10000, type = "int", explanation = "max number of characters a comment can have. default is 500", example = 500, oob_error = "the max comment length should be between 10 and 10000" } -post_lang = { default = "", optional = false, explanation = "The language you would like to translate to - leave blank for none.", example = "es-cr"} +post_lang = { default = "", optional = true, explanation = "The language you would like to translate to.", example = "es-cr"} [settings] allow_nsfw = { optional = false, type = "bool", default = false, example = false, options = [ @@ -30,7 +30,7 @@ theme = { optional = false, default = "light", example = "dark", options = [ ], explanation = "sets the Reddit theme, either LIGHT or DARK" } times_to_run = { optional = false, default = 1, example = 2, explanation = "used if you want to run multiple times. set to an int e.g. 4 or 29 or 1", type = "int", nmin = 1, oob_error = "It's very hard to run something less than once." } opacity = { optional = false, default = 0.9, example = 0.8, explanation = "Sets the opacity of the comments when overlayed over the background", type = "float", nmin = 0, nmax = 1, oob_error = "The opacity HAS to be between 0 and 1", input_error = "The opacity HAS to be a decimal number between 0 and 1" } -storymode = { optional = false, type = "bool", default = false, example = false, options = [ +storymode = { optional = true, type = "bool", default = false, example = false, options = [ true, false, ] } diff --git a/utils/console.py b/utils/console.py index b8faa80..1ffa11c 100644 --- a/utils/console.py +++ b/utils/console.py @@ -54,7 +54,7 @@ def handle_input( + "\n[green]This is an optional value. Do you want to skip it? (y/n)" ) if input().casefold().startswith("y"): - return None + return default if default is not NotImplemented else "" if default is not NotImplemented: console.print( "[green]" diff --git a/utils/settings.py b/utils/settings.py index c77e4ae..afd2ec0 100755 --- a/utils/settings.py +++ b/utils/settings.py @@ -23,11 +23,10 @@ def crawl(obj: dict, func=lambda x, y: print(x, y, end="\n"), path: list = []): def check(value, checks, name): - + def get_check_value(key, default_result): + return checks[key] if key in checks else default_result incorrect = False if value == {}: - if skip_opt and "optional" in checks and checks["optional"] is True: - return None incorrect = True if not incorrect and "type" in checks: try: @@ -96,20 +95,16 @@ def check(value, checks, name): + "[#C0CAF5 bold]" + str(name) + "[#F7768E bold]=", - extra_info=checks["explanation"] if "explanation" in checks else "", - check_type=eval(checks["type"]) if "type" in checks else False, - default=checks["default"] if "default" in checks else NotImplemented, - match=checks["regex"] if "regex" in checks else "", - err_message=checks["input_error"] - if "input_error" in checks - else "Incorrect input", - nmin=checks["nmin"] if "nmin" in checks else None, - nmax=checks["nmax"] if "nmax" in checks else None, - oob_error=checks["oob_error"] - if "oob_error" in checks - else "Input out of bounds(Value too high/low/long/short)", - options=checks["options"] if "options" in checks else None, - optional=checks["optional"] if "optional" in checks else False, + extra_info=get_check_value("explanation", ""), + check_type=eval(get_check_value("type", "False")), + default=get_check_value("default", NotImplemented), + match=get_check_value("regex", ""), + err_message=get_check_value("input_error", "Incorrect input"), + nmin=get_check_value("nmin", None), + nmax=get_check_value("nmax", None), + oob_error=get_check_value("oob_error", "Input out of bounds(Value too high/low/long/short)"), + options=get_check_value("options", None), + optional=get_check_value("optional", False), ) return value @@ -125,8 +120,6 @@ def crawl_and_check(obj: dict, path: list, checks: dict = {}, name=""): def check_vars(path, checks): global config - global skip_opt - skip_opt = "skip_opt" in config crawl_and_check(config, path, checks) @@ -185,7 +178,6 @@ If you see any prompts, that means that you have unset/incorrectly set variables """ ) crawl(template, check_vars) - config["skip_opt"] = True with open(config_file, "w") as f: toml.dump(config, f) return config From 4ed5c58ab264cebac53756d7b21d2e38562f8523 Mon Sep 17 00:00:00 2001 From: Callum Leslie Date: Wed, 6 Jul 2022 14:42:16 +0100 Subject: [PATCH 062/126] style: formatted with black --- utils/settings.py | 5 ++++- video_creation/final_video.py | 17 ++++++++--------- 2 files changed, 12 insertions(+), 10 deletions(-) diff --git a/utils/settings.py b/utils/settings.py index afd2ec0..07cebd4 100755 --- a/utils/settings.py +++ b/utils/settings.py @@ -25,6 +25,7 @@ def crawl(obj: dict, func=lambda x, y: print(x, y, end="\n"), path: list = []): def check(value, checks, name): def get_check_value(key, default_result): return checks[key] if key in checks else default_result + incorrect = False if value == {}: incorrect = True @@ -102,7 +103,9 @@ def check(value, checks, name): err_message=get_check_value("input_error", "Incorrect input"), nmin=get_check_value("nmin", None), nmax=get_check_value("nmax", None), - oob_error=get_check_value("oob_error", "Input out of bounds(Value too high/low/long/short)"), + oob_error=get_check_value( + "oob_error", "Input out of bounds(Value too high/low/long/short)" + ), options=get_check_value("options", None), optional=get_check_value("optional", False), ) diff --git a/video_creation/final_video.py b/video_creation/final_video.py index 5938235..1c5e781 100755 --- a/video_creation/final_video.py +++ b/video_creation/final_video.py @@ -26,15 +26,14 @@ console = Console() W, H = 1080, 1920 -def name_normalize( - name: str -) -> str: - name = re.sub(r'[?\\"%*:|<>]', '', name) - name = re.sub(r'( [w,W]\s?\/\s?[o,O,0])', r' without', name) - name = re.sub(r'( [w,W]\s?\/)', r' with', name) - name = re.sub(r'([0-9]+)\s?\/\s?([0-9]+)', r'\1 of \2', name) - name = re.sub(r'(\w+)\s?\/\s?(\w+)', r'\1 or \2', name) - name = re.sub(r'\/', r'', name) + +def name_normalize(name: str) -> str: + name = re.sub(r'[?\\"%*:|<>]', "", name) + name = re.sub(r"( [w,W]\s?\/\s?[o,O,0])", r" without", name) + name = re.sub(r"( [w,W]\s?\/)", r" with", name) + name = re.sub(r"([0-9]+)\s?\/\s?([0-9]+)", r"\1 of \2", name) + name = re.sub(r"(\w+)\s?\/\s?(\w+)", r"\1 or \2", name) + name = re.sub(r"\/", r"", name) return name From cebbbf72cb2a75170e5ea24bd12550cc189c1d5e Mon Sep 17 00:00:00 2001 From: Tomovic Date: Wed, 6 Jul 2022 16:19:36 +0200 Subject: [PATCH 063/126] Added translation for filename TOML compatible --- video_creation/final_video.py | 11 ++++++++++- 1 file changed, 10 insertions(+), 1 deletion(-) diff --git a/video_creation/final_video.py b/video_creation/final_video.py index 5938235..b56d231 100755 --- a/video_creation/final_video.py +++ b/video_creation/final_video.py @@ -4,6 +4,7 @@ import os import re from os.path import exists from typing import Dict +import translators as ts from moviepy.editor import ( VideoFileClip, @@ -35,7 +36,15 @@ def name_normalize( name = re.sub(r'([0-9]+)\s?\/\s?([0-9]+)', r'\1 of \2', name) name = re.sub(r'(\w+)\s?\/\s?(\w+)', r'\1 or \2', name) name = re.sub(r'\/', r'', name) - return name + + lang = settings.config["reddit"]["thread"]["post_lang"] + if lang != "": + print_substep("Translating filename...") + translated_name = ts.google(name, to_language = lang) + return translated_name + + else: + return name def make_final_video( From 71eca6991215e1e8538c03c789b4893bacc0eef3 Mon Sep 17 00:00:00 2001 From: Callum Leslie Date: Wed, 6 Jul 2022 20:08:45 +0000 Subject: [PATCH 064/126] fix: handle non value for post lang --- video_creation/final_video.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/video_creation/final_video.py b/video_creation/final_video.py index 1080f66..4f73cf8 100755 --- a/video_creation/final_video.py +++ b/video_creation/final_video.py @@ -37,7 +37,7 @@ def name_normalize(name: str) -> str: name = re.sub(r"\/", r"", name) lang = settings.config["reddit"]["thread"]["post_lang"] - if lang != "": + if lang: print_substep("Translating filename...") translated_name = ts.google(name, to_language = lang) return translated_name From 4d7db77fff3097451a937ced47bf88d3655ec2e5 Mon Sep 17 00:00:00 2001 From: Callum Leslie Date: Wed, 6 Jul 2022 23:14:43 +0100 Subject: [PATCH 065/126] style: reformat with python-black --- video_creation/final_video.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/video_creation/final_video.py b/video_creation/final_video.py index 4f73cf8..42f13b4 100755 --- a/video_creation/final_video.py +++ b/video_creation/final_video.py @@ -35,11 +35,11 @@ def name_normalize(name: str) -> str: name = re.sub(r"([0-9]+)\s?\/\s?([0-9]+)", r"\1 of \2", name) name = re.sub(r"(\w+)\s?\/\s?(\w+)", r"\1 or \2", name) name = re.sub(r"\/", r"", name) - + lang = settings.config["reddit"]["thread"]["post_lang"] if lang: print_substep("Translating filename...") - translated_name = ts.google(name, to_language = lang) + translated_name = ts.google(name, to_language=lang) return translated_name else: From b82459166b9810c9aaeae627e1d3cac5f8fc891c Mon Sep 17 00:00:00 2001 From: Jason Date: Wed, 6 Jul 2022 19:17:55 -0400 Subject: [PATCH 066/126] docs: improved examples in .config.template.toml fix: mutability issue in settings.py style: added autocomplete support for the settings.config var style: fixed autocomplete issues overall --- .config.template.toml | 8 ++--- .github/workflows/stale.yml | 40 ------------------------- utils/settings.py | 13 ++++---- utils/videos.py | 4 +-- video_creation/screenshot_downloader.py | 4 +-- video_creation/voices.py | 2 +- 6 files changed, 15 insertions(+), 56 deletions(-) delete mode 100644 .github/workflows/stale.yml diff --git a/.config.template.toml b/.config.template.toml index e1089e3..1bde264 100644 --- a/.config.template.toml +++ b/.config.template.toml @@ -1,7 +1,7 @@ [reddit.creds] client_id = { optional = false, nmin = 12, nmax = 30, explanation = "the ID of your Reddit app of SCRIPT type", example = "fFAGRNJru1FTz70BzhT3Zg", regex = "^[-a-zA-Z0-9._~+/]+=*$", input_error = "The client ID can only contain printable characters.", oob_error = "The ID should be over 12 and under 30 characters, double check your input." } client_secret = { optional = false, nmin = 20, nmax = 40, explanation = "the SECRET of your Reddit app of SCRIPT type", example = "fFAGRNJru1FTz70BzhT3Zg", regex = "^[-a-zA-Z0-9._~+/]+=*$", input_error = "The client ID can only contain printable characters.", oob_error = "The secret should be over 20 and under 40 characters, double check your input." } -username = { optional = false, nmin = 3, nmax = 20, explanation = "the username of your reddit account", example = "asdfghjkl", regex = "^[-_0-9a-zA-Z]+$", oob_error = "A username HAS to be between 3 and 20 characters" } +username = { optional = false, nmin = 3, nmax = 20, explanation = "the username of your reddit account", example = "JasonLovesDoggo", regex = "^[-_0-9a-zA-Z]+$", oob_error = "A username HAS to be between 3 and 20 characters" } password = { optional = false, nmin = 8, explanation = "the password of your reddit account", example = "fFAGRNJru1FTz70BzhT3Zg", oob_error = "Password too short" } 2fa = { optional = true, type = "bool", options = [ true, @@ -15,7 +15,7 @@ random = { optional = true, options = [ false, ], default = false, type = "bool", explanation = "If set to no, it will ask you a thread link to extract the thread, if yes it will randomize it. Default: 'False'", example = "True" } subreddit = { optional = false, regex = "[_0-9a-zA-Z]+$", nmin = 3, nmax = 21, explanation = "what subreddit to pull posts from, the name of the sub, not the URL", example = "AskReddit", oob_error = "A subreddit name HAS to be between 3 and 20 characters" } -post_id = { optional = true, default = "", regex = "^((?!://|://).)*$", explanation = "Used if you want to use a specific post.", example = "urdtfx" } +post_id = { optional = true, default = "", regex = "^((?!://|://)[+a-zA-Z])*$", explanation = "Used if you want to use a specific post.", example = "urdtfx" } max_comment_length = { default = 500, optional = false, nmin = 10, nmax = 10000, type = "int", explanation = "max number of characters a comment can have. default is 500", example = 500, oob_error = "the max comment length should be between 10 and 10000" } post_lang = { default = "", optional = true, explanation = "The language you would like to translate to.", example = "es-cr"} @@ -24,7 +24,7 @@ allow_nsfw = { optional = false, type = "bool", default = false, example = false true, false, ], explanation = "Whether to allow NSFW content, True or False" } -theme = { optional = false, default = "light", example = "dark", options = [ +theme = { optional = false, default = "dark", example = "light", options = [ "dark", "light", ], explanation = "sets the Reddit theme, either LIGHT or DARK" } @@ -37,7 +37,7 @@ storymode = { optional = true, type = "bool", default = false, example = false, [settings.tts] +choice = { optional = false, default = "", options = [ "streamlabspolly", "tiktok", "googletranslate", "awspolly",], example = "streamlabspolly", explanation = "The backend used for TTS generation. This can be left blank and you will be prompted to choose at runtime." } aws_polly_voice = { optional = false, default = "Matthew", example = "Matthew", explanation = "The voice used for AWS Polly" } streamlabs_polly_voice = { optional = false, default = "Matthew", example = "Matthew", explanation = "The voice used for Streamlabs Polly" } tiktok_voice = { optional = false, default = "en_us_006", example = "en_us_006", explanation = "The voice used for TikTok TTS" } -choice = { optional = false, default = "", options = [ "streamlabspolly", "tiktok", "googletranslate", "awspolly", ""], example = "streamlabspolly", explanation = "The backend used for TTS generation. This can be left blank and you will be prompted to choose at runtime." } diff --git a/.github/workflows/stale.yml b/.github/workflows/stale.yml deleted file mode 100644 index 40f2245..0000000 --- a/.github/workflows/stale.yml +++ /dev/null @@ -1,40 +0,0 @@ -name: 'Stale issue handler' -on: - workflow_dispatch: - schedule: - - cron: '0 0 * * *' - -jobs: - stale: - runs-on: ubuntu-latest - steps: - - uses: actions/stale@main - id: stale-issue - name: stale-issue - with: - stale-issue-message: 'This issue is stale because it has been open 7 days with no activity. Remove stale label or comment, or this will be closed in 10 days.' - close-issue-message: 'Issue closed due to being stale. Please reopen if issue persists in latest version.' - days-before-stale: 7 - days-before-close: 10 - stale-issue-label: 'stale' - close-issue-label: 'outdated' - exempt-issue-labels: 'enhancement,keep,blocked' - exempt-all-issue-milestones: true - operations-per-run: 300 - remove-stale-when-updated: true - - - uses: actions/stale@main - id: stale-pr - name: stale-pr - with: - stale-pr-message: 'This pull request is stale as it has been open for 7 days with no activity. Remove stale label or comment, or this will be closed in 10 days.' - close-pr-message: 'Pull request closed due to being stale.' - days-before-stale: 7 - days-before-close: 10 - close-pr-label: 'outdated' - stale-pr-label: 'stale' - exempt-pr-labels: 'keep,blocked,before next release,after next release' - exempt-all-pr-milestones: true - operations-per-run: 300 - remove-stale-when-updated: true - diff --git a/utils/settings.py b/utils/settings.py index 07cebd4..7c2b05d 100755 --- a/utils/settings.py +++ b/utils/settings.py @@ -1,5 +1,4 @@ #!/usr/bin/env python -# import os import toml from rich.console import Console import re @@ -8,13 +7,13 @@ from typing import Tuple, Dict from utils.console import handle_input -# from console import handle_input - console = Console() +config = dict # autocomplete - -def crawl(obj: dict, func=lambda x, y: print(x, y, end="\n"), path: list = []): +def crawl(obj: dict, func=lambda x, y: print(x, y, end="\n"), path=None): + if path is None: # path Default argument value is mutable + path = [] for key in obj.keys(): if type(obj[key]) is dict: crawl(obj[key], func, path + [key]) @@ -138,7 +137,7 @@ def check_toml(template_file, config_file) -> Tuple[bool, Dict]: return False try: config = toml.load(config_file) - except (toml.TomlDecodeError): + except toml.TomlDecodeError: console.print( f"""[blue]Couldn't read {config_file}. Overwrite it?(y/n)""" @@ -155,7 +154,7 @@ Overwrite it?(y/n)""" f"[red bold]Failed to overwrite {config_file}. Giving up.\nSuggestion: check {config_file} permissions for the user." ) return False - except (FileNotFoundError): + except FileNotFoundError: console.print( f"""[blue]Couldn't find {config_file} Creating it now.""" diff --git a/utils/videos.py b/utils/videos.py index 38184e4..4e36729 100755 --- a/utils/videos.py +++ b/utils/videos.py @@ -15,10 +15,10 @@ def check_done( """Checks if the chosen post has already been generated Args: - redditobj (Dict[str]): Reddit object gotten from reddit/subreddit.py + redditobj (Submission): Reddit object gotten from reddit/subreddit.py Returns: - Dict[str]|None: Reddit object in args + Submission|None: Reddit object in args """ with open( "./video_creation/data/videos.json", "r", encoding="utf-8" diff --git a/video_creation/screenshot_downloader.py b/video_creation/screenshot_downloader.py index efc48bd..9dbb3d8 100644 --- a/video_creation/screenshot_downloader.py +++ b/video_creation/screenshot_downloader.py @@ -20,8 +20,8 @@ def download_screenshots_of_reddit_posts(reddit_object: dict, screenshot_num: in """Downloads screenshots of reddit posts as seen on the web. Downloads to assets/temp/png Args: - reddit_object (Dict[str]): Reddit object received from reddit/subreddit.py - screenshot_num (int): Number of screenshots to downlaod + reddit_object (Dict): Reddit object received from reddit/subreddit.py + screenshot_num (int): Number of screenshots to download """ print_step("Downloading screenshots of reddit posts...") diff --git a/video_creation/voices.py b/video_creation/voices.py index 4bbd5d7..e6e1045 100644 --- a/video_creation/voices.py +++ b/video_creation/voices.py @@ -27,7 +27,7 @@ def save_text_to_mp3(reddit_obj) -> Tuple[int, int]: """Saves text to MP3 files. Args: - reddit_obj (dict[str]): Reddit object received from reddit API in reddit/subreddit.py + reddit_obj (): Reddit object received from reddit API in reddit/subreddit.py Returns: tuple[int,int]: (total length of the audio, the number of comments audio was generated for) From e3c74a2b86ed651d1cdb0aecf190e4d4e2863678 Mon Sep 17 00:00:00 2001 From: Jason Date: Wed, 6 Jul 2022 19:18:56 -0400 Subject: [PATCH 067/126] fixed local issue --- .github/workflows/stale.yml | 39 +++++++++++++++++++++++++++++++++++++ 1 file changed, 39 insertions(+) create mode 100644 .github/workflows/stale.yml diff --git a/.github/workflows/stale.yml b/.github/workflows/stale.yml new file mode 100644 index 0000000..b2f6cac --- /dev/null +++ b/.github/workflows/stale.yml @@ -0,0 +1,39 @@ +name: 'Stale issue handler' +on: + workflow_dispatch: + schedule: + - cron: '0 0 * * *' + +jobs: + stale: + runs-on: ubuntu-latest + steps: + - uses: actions/stale@main + id: stale-issue + name: stale-issue + with: + stale-issue-message: 'This issue is stale because it has been open 7 days with no activity. Remove stale label or comment, or this will be closed in 10 days.' + close-issue-message: 'Issue closed due to being stale. Please reopen if issue persists in latest version.' + days-before-stale: 7 + days-before-close: 10 + stale-issue-label: 'stale' + close-issue-label: 'outdated' + exempt-issue-labels: 'enhancement,keep,blocked' + exempt-all-issue-milestones: true + operations-per-run: 300 + remove-stale-when-updated: true + + - uses: actions/stale@main + id: stale-pr + name: stale-pr + with: + stale-pr-message: 'This pull request is stale as it has been open for 7 days with no activity. Remove stale label or comment, or this will be closed in 10 days.' + close-pr-message: 'Pull request closed due to being stale.' + days-before-stale: 7 + days-before-close: 10 + close-pr-label: 'outdated' + stale-pr-label: 'stale' + exempt-pr-labels: 'keep,blocked,before next release,after next release' + exempt-all-pr-milestones: true + operations-per-run: 300 + remove-stale-when-updated: true From a9f0449ddd1408ff90f683190b8d298d2016f826 Mon Sep 17 00:00:00 2001 From: Jason <66544866+JasonLovesDoggo@users.noreply.github.com> Date: Wed, 6 Jul 2022 22:26:06 -0400 Subject: [PATCH 068/126] Delete .env.template --- .env.template | 91 --------------------------------------------------- 1 file changed, 91 deletions(-) delete mode 100644 .env.template diff --git a/.env.template b/.env.template deleted file mode 100644 index 1cb62cb..0000000 --- a/.env.template +++ /dev/null @@ -1,91 +0,0 @@ - -REDDIT_CLIENT_ID="" #fFAGRNJru1FTz70BzhT3Zg -#EXPLANATION the ID of your Reddit app of SCRIPT type -#RANGE 12:30 -#MATCH_REGEX [-a-zA-Z0-9._~+/]+=*$ -#OOB_ERROR The ID should be over 12 and under 30 characters, double check your input. - -REDDIT_CLIENT_SECRET="" #fFAGRNJru1FTz70BzhT3Zg -#EXPLANATION the SECRET of your Reddit app of SCRIPT type -#RANGE 20:40 -#MATCH_REGEX [-a-zA-Z0-9._~+/]+=*$ -#OOB_ERROR The secret should be over 20 and under 40 characters, double check your input. - -REDDIT_USERNAME="" #asdfghjkl -#EXPLANATION the username of your reddit account -#RANGE 3:20 -#MATCH_REGEX [-_0-9a-zA-Z]+$ -#OOB_ERROR A username HAS to be between 3 and 20 characters - -REDDIT_PASSWORD="" #fFAGRNJru1FTz70BzhT3Zg -#EXPLANATION the password of your reddit account -#RANGE 8:None -#OOB_ERROR Password too short - -#OPTIONAL -RANDOM_THREAD="no" -# If set to no, it will ask you a thread link to extract the thread, if yes it will randomize it. Default: "no" - -REDDIT_2FA="" #no -#MATCH_REGEX ^(yes|no) -#EXPLANATION Whether you have Reddit 2FA enabled, Valid options are "yes" and "no" - -SUBREDDIT="AskReddit" -#EXPLANATION what subreddit to pull posts from, the name of the sub, not the URL -#RANGE 3:20 -#MATCH_REGEX [_0-9a-zA-Z]+$ -#OOB_ERROR A subreddit name HAS to be between 3 and 20 characters - -ALLOW_NSFW="False" -#EXPLANATION Whether to allow NSFW content, True or False -#MATCH_REGEX ^(True|False)$ - -POST_ID="" -#MATCH_REGEX ^((?!://|://).)*$ -#EXPLANATION Used if you want to use a specific post. example of one is urdtfx - -THEME="LIGHT" #dark -#EXPLANATION sets the Reddit theme, either LIGHT or DARK -#MATCH_REGEX ^(dark|light|DARK|LIGHT)$ - -TIMES_TO_RUN="" #2 -#EXPLANATION used if you want to run multiple times. set to an int e.g. 4 or 29 and leave blank for 1 - -MAX_COMMENT_LENGTH="500" #500 -#EXPLANATION max number of characters a comment can have. default is 500 -#RANGE 0:10000 -#MATCH_TYPE int -#OOB_ERROR the max comment length should be between 0 and 10000 - -OPACITY="1" #.8 -#EXPLANATION Sets the opacity of the comments when overlayed over the background -#RANGE 0:1 -#MATCH_TYPE float -#OOB_ERROR The opacity HAS to be between 0 and 1 - -# If you want to translate the comments to another language, set the language code here. -# If empty, no translation will be done. -POSTLANG="" -#EXPLANATION Activates the translation feature, set the language code for translate or leave blank - -TTSCHOICE="Polly" -#EXPLANATION the backend used for TTS. Without anything specified, the user will be prompted to choose one. -# IMPORTANT NOTE: if you use translate, you need to set this to googletranslate or tiktok and use custom voice in your language - -STREAMLABS_VOICE="Joanna" -#EXPLANATION Sets the voice for the Streamlabs Polly TTS Engine. Check the file for more information on different voices. - -AWS_VOICE="Joanna" -#EXPLANATION Sets the voice for the AWS Polly TTS Engine. Check the file for more information on different voices. - -TIKTOK_VOICE="en_us_006" -#EXPLANATION Sets the voice for the TikTok TTS Engine. Check the file for more information on different voices. - -#OPTIONAL -BackgroundChoice="minecraft" -# EXPLANATION Sets the background for the video. Current available option : (minecraft,gta,rocket-league,motor-gta), but you can add other video easily (1080p). Check the file for more information on different background. - -#OPTIONAL -STORYMODE="False" -# IN-PROGRESS - not yet implemented - From 0b6e6a4c140f068e113d65431598ca9b434a197d Mon Sep 17 00:00:00 2001 From: Jason Date: Wed, 6 Jul 2022 22:29:47 -0400 Subject: [PATCH 069/126] added toml config for #693 --- .config.template.toml | 31 +++++++++++++------------------ 1 file changed, 13 insertions(+), 18 deletions(-) diff --git a/.config.template.toml b/.config.template.toml index 1bde264..8775357 100644 --- a/.config.template.toml +++ b/.config.template.toml @@ -3,41 +3,36 @@ client_id = { optional = false, nmin = 12, nmax = 30, explanation = "the ID of y client_secret = { optional = false, nmin = 20, nmax = 40, explanation = "the SECRET of your Reddit app of SCRIPT type", example = "fFAGRNJru1FTz70BzhT3Zg", regex = "^[-a-zA-Z0-9._~+/]+=*$", input_error = "The client ID can only contain printable characters.", oob_error = "The secret should be over 20 and under 40 characters, double check your input." } username = { optional = false, nmin = 3, nmax = 20, explanation = "the username of your reddit account", example = "JasonLovesDoggo", regex = "^[-_0-9a-zA-Z]+$", oob_error = "A username HAS to be between 3 and 20 characters" } password = { optional = false, nmin = 8, explanation = "the password of your reddit account", example = "fFAGRNJru1FTz70BzhT3Zg", oob_error = "Password too short" } -2fa = { optional = true, type = "bool", options = [ - true, - false, +2fa = { optional = true, type = "bool", options = [true, + false, ], default = false, explanation = "Whether you have Reddit 2FA enabled, Valid options are True and False", example = true } [reddit.thread] -random = { optional = true, options = [ - true, - false, +random = { optional = true, options = [true, + false, ], default = false, type = "bool", explanation = "If set to no, it will ask you a thread link to extract the thread, if yes it will randomize it. Default: 'False'", example = "True" } subreddit = { optional = false, regex = "[_0-9a-zA-Z]+$", nmin = 3, nmax = 21, explanation = "what subreddit to pull posts from, the name of the sub, not the URL", example = "AskReddit", oob_error = "A subreddit name HAS to be between 3 and 20 characters" } post_id = { optional = true, default = "", regex = "^((?!://|://)[+a-zA-Z])*$", explanation = "Used if you want to use a specific post.", example = "urdtfx" } max_comment_length = { default = 500, optional = false, nmin = 10, nmax = 10000, type = "int", explanation = "max number of characters a comment can have. default is 500", example = 500, oob_error = "the max comment length should be between 10 and 10000" } -post_lang = { default = "", optional = true, explanation = "The language you would like to translate to.", example = "es-cr"} +post_lang = { default = "", optional = true, explanation = "The language you would like to translate to.", example = "es-cr" } [settings] -allow_nsfw = { optional = false, type = "bool", default = false, example = false, options = [ - true, - false, +allow_nsfw = { optional = false, type = "bool", default = false, example = false, options = [true, + false, ], explanation = "Whether to allow NSFW content, True or False" } -theme = { optional = false, default = "dark", example = "light", options = [ - "dark", - "light", +theme = { optional = false, default = "dark", example = "light", options = ["dark", + "light", ], explanation = "sets the Reddit theme, either LIGHT or DARK" } times_to_run = { optional = false, default = 1, example = 2, explanation = "used if you want to run multiple times. set to an int e.g. 4 or 29 or 1", type = "int", nmin = 1, oob_error = "It's very hard to run something less than once." } opacity = { optional = false, default = 0.9, example = 0.8, explanation = "Sets the opacity of the comments when overlayed over the background", type = "float", nmin = 0, nmax = 1, oob_error = "The opacity HAS to be between 0 and 1", input_error = "The opacity HAS to be a decimal number between 0 and 1" } -storymode = { optional = true, type = "bool", default = false, example = false, options = [ - true, - false, +storymode = { optional = true, type = "bool", default = false, example = false, options = [true, + false, ] } - +background_choice = { optional = true, default = "minecraft", example = "minecraft", options = ["minecraft", "gta", "rocket-league", "motor-gta"], explanation = "Sets the background for the video" } [settings.tts] -choice = { optional = false, default = "", options = [ "streamlabspolly", "tiktok", "googletranslate", "awspolly",], example = "streamlabspolly", explanation = "The backend used for TTS generation. This can be left blank and you will be prompted to choose at runtime." } +choice = { optional = false, default = "", options = ["streamlabspolly", "tiktok", "googletranslate", "awspolly", ], example = "streamlabspolly", explanation = "The backend used for TTS generation. This can be left blank and you will be prompted to choose at runtime." } aws_polly_voice = { optional = false, default = "Matthew", example = "Matthew", explanation = "The voice used for AWS Polly" } streamlabs_polly_voice = { optional = false, default = "Matthew", example = "Matthew", explanation = "The voice used for Streamlabs Polly" } tiktok_voice = { optional = false, default = "en_us_006", example = "en_us_006", explanation = "The voice used for TikTok TTS" } From bd52b4fe8b74c58703aca049ef43a7f50f7389d0 Mon Sep 17 00:00:00 2001 From: Jason Date: Wed, 6 Jul 2022 22:58:35 -0400 Subject: [PATCH 070/126] some changes to see merge conflicts --- main.py | 5 ++-- video_creation/background.py | 48 +++++++++++++++++++++++++++++++++++ video_creation/final_video.py | 22 +++++++--------- 3 files changed, 60 insertions(+), 15 deletions(-) diff --git a/main.py b/main.py index c7079d5..568c17e 100755 --- a/main.py +++ b/main.py @@ -8,12 +8,12 @@ from utils.console import print_markdown, print_step from utils import settings # from utils.checker import envUpdate -from video_creation.background import download_background, chop_background_video +from video_creation.background import download_background, chop_background_video, get_background_config from video_creation.final_video import make_final_video from video_creation.screenshot_downloader import download_screenshots_of_reddit_posts from video_creation.voices import save_text_to_mp3 -VERSION = "2.2.2" +VERSION = "2.2.8" print( """ ██████╗ ███████╗██████╗ ██████╗ ██╗████████╗ ██╗ ██╗██╗██████╗ ███████╗ ██████╗ ███╗ ███╗ █████╗ ██╗ ██╗███████╗██████╗ @@ -36,6 +36,7 @@ def main(POST_ID=None): reddit_object = get_subreddit_threads(POST_ID) length, number_of_comments = save_text_to_mp3(reddit_object) length = math.ceil(length) + bg_config = get_background_config() download_screenshots_of_reddit_posts(reddit_object, number_of_comments) download_background() credit = chop_background_video(length) diff --git a/video_creation/background.py b/video_creation/background.py index 494c7d2..42f6044 100644 --- a/video_creation/background.py +++ b/video_creation/background.py @@ -8,8 +8,56 @@ from moviepy.editor import VideoFileClip from moviepy.video.io.ffmpeg_tools import ffmpeg_extract_subclip from pytube import YouTube +from utils import settings from utils.console import print_step, print_substep +# Supported Background. Can add/remove background video here.... +# - : key -> used as keyword for TOML file. value -> background configuration +# Format (value): +# 1. Youtube URI +# 2. filename +# 3. Citation (owner of the video) +# 4. Position of image clips in the background. See moviepy reference for more information. (https://zulko.github.io/moviepy/ref/VideoClip/VideoClip.html#moviepy.video.VideoClip.VideoClip.set_position) +background_options = { + "motor-gta": ( # Motor-GTA Racing + "https://www.youtube.com/watch?v=vw5L4xCPy9Q", + "bike-parkour-gta.mp4", + "Achy Gaming", + lambda t: ('center', 480 + t) + ), + "rocket-league": ( # Rocket League + "https://www.youtube.com/watch?v=2X9QGY__0II", + "rocket_league.mp4", + "Orbital Gameplay", + "top" + ), + "minecraft": ( # Minecraft parkour + "https://www.youtube.com/watch?v=n_Dv4JMiwK8", + "parkour.mp4", + "bbswitzer", + "center" + ), + "gta": ( # GTA Stunt Race + "https://www.youtube.com/watch?v=qGa9kWREOnE", + "gta-stunt-race.mp4", + "Achy Gaming", + lambda t: ('center', 480 + t) + ) +} +def get_background_config(): + """Fetch the background/s configuration""" + try: + choice = str(settings.config['settings']['background_choice']).casefold() + except AttributeError: + print_substep("No background selected. Picking random background'") + choice = None + + # Handle default / not supported background using default option. + # Default : pick random from supported background. + if not choice or choice not in background_options: + choice = random.choice(list(background_options.keys())) + + return background_options[choice] def get_start_and_end_times(video_length: int, length_of_clip: int) -> Tuple[int, int]: """Generates a random interval of time to be used as the background of the video. diff --git a/video_creation/final_video.py b/video_creation/final_video.py index 42f13b4..c06598e 100755 --- a/video_creation/final_video.py +++ b/video_creation/final_video.py @@ -3,7 +3,7 @@ import multiprocessing import os import re from os.path import exists -from typing import Dict +from typing import Dict, Tuple, Any import translators as ts from moviepy.editor import ( @@ -32,7 +32,7 @@ def name_normalize(name: str) -> str: name = re.sub(r'[?\\"%*:|<>]', "", name) name = re.sub(r"( [w,W]\s?\/\s?[o,O,0])", r" without", name) name = re.sub(r"( [w,W]\s?\/)", r" with", name) - name = re.sub(r"([0-9]+)\s?\/\s?([0-9]+)", r"\1 of \2", name) + name = re.sub(r"(\d+)\s?\/\s?(\d+)", r"\1 of \2", name) name = re.sub(r"(\w+)\s?\/\s?(\w+)", r"\1 or \2", name) name = re.sub(r"\/", r"", name) @@ -46,15 +46,13 @@ def name_normalize(name: str) -> str: return name -def make_final_video( - number_of_clips: int, length: int, reddit_obj: dict, background_credit: str -): +def make_final_video(number_of_clips: int, length: int, reddit_obj: dict, background_config: Tuple[str, str, str, Any]): """Gathers audio clips, gathers all screenshots, stitches them together and saves the final video to assets/temp - Args: number_of_clips (int): Index to end at when going through the screenshots length (int): Length of the video reddit_obj (dict): The reddit object that contains the posts to read. + background_config Tuple[str, str, str, Any]: The background config to use. """ print_step("Creating the final video 🎥") VideoFileClip.reW = lambda clip: clip.resize(width=W) @@ -85,7 +83,6 @@ def make_final_video( 0, ImageClip("assets/temp/png/title.png") .set_duration(audio_clips[0].duration) - .set_position("center") .resize(width=W - 100) .set_opacity(new_opacity), ) @@ -94,7 +91,6 @@ def make_final_video( image_clips.append( ImageClip(f"assets/temp/png/comment_{i}.png") .set_duration(audio_clips[i + 1].duration) - .set_position("center") .resize(width=W - 100) .set_opacity(new_opacity) ) @@ -109,9 +105,9 @@ def make_final_video( # .set_opacity(float(opacity)), # ) # else: - image_concat = concatenate_videoclips(image_clips).set_position( - ("center", "center") - ) + img_clip_pos = background_config[3] + image_concat = concatenate_videoclips( + image_clips).set_position(img_clip_pos) image_concat.audio = audio_composite final = CompositeVideoClip([background_clip, image_concat]) title = re.sub(r"[^\w\s-]", "", reddit_obj["thread_title"]) @@ -120,7 +116,7 @@ def make_final_video( filename = f"{name_normalize(title)}.mp4" subreddit = settings.config["reddit"]["thread"]["subreddit"] - save_data(filename, title, idx, background_credit) + save_data(filename, title, idx, background_config[2]) if not exists(f"./results/{subreddit}"): print_substep("The results folder didn't exist so I made it") @@ -148,5 +144,5 @@ def make_final_video( print_substep("See result in the results folder!") print_step( - f'Reddit title: {reddit_obj["thread_title"]} \n Background Credit: {background_credit}' + f'Reddit title: {reddit_obj["thread_title"]} \n Background Credit: {background_config[2]}' ) From 9ae01b7c1947caa5044a0acce3f640bee4716c9e Mon Sep 17 00:00:00 2001 From: Jason Date: Wed, 6 Jul 2022 23:15:56 -0400 Subject: [PATCH 071/126] fixed merge conflicts and cleared up some old existing .env referances chore: updated version num style: improved IDE autocorrection support chore: simplified the regex in func name_normalize --- TTS/aws_polly.py | 2 +- TTS/streamlabs_polly.py | 2 +- main.py | 2 +- utils/videos.py | 4 ++-- video_creation/background.py | 25 ++----------------------- video_creation/final_video.py | 6 +++--- 6 files changed, 10 insertions(+), 31 deletions(-) diff --git a/TTS/aws_polly.py b/TTS/aws_polly.py index bf8ec1e..0ff0d74 100644 --- a/TTS/aws_polly.py +++ b/TTS/aws_polly.py @@ -37,7 +37,7 @@ class AWSPolly: else: if not settings.config["settings"]["tts"]["aws_polly_voice"]: return ValueError( - f"Please set the environment variable AWS_VOICE to a valid voice. options are: {voices}" + f"Please set the TOML variable AWS_VOICE to a valid voice. options are: {voices}" ) voice = str( settings.config["settings"]["tts"]["aws_polly_voice"] diff --git a/TTS/streamlabs_polly.py b/TTS/streamlabs_polly.py index e9e6358..d4ddcb4 100644 --- a/TTS/streamlabs_polly.py +++ b/TTS/streamlabs_polly.py @@ -37,7 +37,7 @@ class StreamlabsPolly: else: if not settings.config["settings"]["tts"]["streamlabs_polly_voice"]: return ValueError( - f"Please set the environment variable STREAMLABS_VOICE to a valid voice. options are: {voices}" + f"Please set the config variable STREAMLABS_VOICE to a valid voice. options are: {voices}" ) voice = str( settings.config["settings"]["tts"]["streamlabs_polly_voice"] diff --git a/main.py b/main.py index 91d9e56..622cd2e 100755 --- a/main.py +++ b/main.py @@ -13,7 +13,7 @@ from video_creation.final_video import make_final_video from video_creation.screenshot_downloader import download_screenshots_of_reddit_posts from video_creation.voices import save_text_to_mp3 -VERSION = "2.2.8" +VERSION = "2.2.9" print( """ ██████╗ ███████╗██████╗ ██████╗ ██╗████████╗ ██╗ ██╗██╗██████╗ ███████╗ ██████╗ ███╗ ███╗ █████╗ ██╗ ██╗███████╗██████╗ diff --git a/utils/videos.py b/utils/videos.py index 4e36729..51143c3 100755 --- a/utils/videos.py +++ b/utils/videos.py @@ -28,7 +28,7 @@ def check_done( if video["id"] == str(redditobj): if settings.config["reddit"]["thread"]["post_id"]: print_step( - "You already have done this video but since it was declared specifically in the .env file the program will continue" + "You already have done this video but since it was declared specifically in the config file the program will continue" ) return redditobj print_step("Getting new post as the current one has already been done") @@ -48,7 +48,7 @@ def save_data(filename: str, reddit_title: str, reddit_id: str, credit: str): with open("./video_creation/data/videos.json", "r+", encoding="utf-8") as raw_vids: done_vids = json.load(raw_vids) if reddit_id in [video["id"] for video in done_vids]: - return # video already done but was specified to continue anyway in the .env file + return # video already done but was specified to continue anyway in the config file payload = { "id": reddit_id, "time": str(int(time.time())), diff --git a/video_creation/background.py b/video_creation/background.py index 4044f15..d9dc056 100644 --- a/video_creation/background.py +++ b/video_creation/background.py @@ -1,11 +1,8 @@ -import random -from os import listdir from pathlib import Path import random from random import randrange from typing import Any, Tuple -from dotenv import load_dotenv from moviepy.editor import VideoFileClip from moviepy.video.io.ffmpeg_tools import ffmpeg_extract_subclip @@ -48,21 +45,7 @@ background_options = { lambda t: ('center', 480 + t) ) } -def get_background_config(): - """Fetch the background/s configuration""" - try: - choice = str(settings.config['settings']['background_choice']).casefold() - except AttributeError: - print_substep("No background selected. Picking random background'") - choice = None - # Handle default / not supported background using default option. - # Default : pick random from supported background. - if not choice or choice not in background_options: - choice = random.choice(list(background_options.keys())) - - return background_options[choice] - def get_start_and_end_times(video_length: int, length_of_clip: int) -> Tuple[int, int]: """Generates a random interval of time to be used as the background of the video. @@ -76,12 +59,10 @@ def get_start_and_end_times(video_length: int, length_of_clip: int) -> Tuple[int random_time = randrange(180, int(length_of_clip) - int(video_length)) return random_time, random_time + video_length - def get_background_config(): """Fetch the background/s configuration""" - load_dotenv() try: - choice = getenv("BackgroundChoice").casefold() + choice = str(settings.config['settings']['background_choice']).casefold() except AttributeError: print_substep("No background selected. Picking random background'") choice = None @@ -123,8 +104,6 @@ def chop_background_video(background_config: Tuple[str, str, str, Any], video_le print_step("Finding a spot in the backgrounds video to chop...✂️") choice = f"{background_config[2]}-{background_config[1]}" - environ["background_credit"] = choice.split("-")[0] - background = VideoFileClip(f"assets/backgrounds/{choice}") @@ -143,4 +122,4 @@ def chop_background_video(background_config: Tuple[str, str, str, Any], video_le new = video.subclip(start_time, end_time) new.write_videofile("assets/temp/background.mp4") print_substep("Background video chopped successfully!", style="bold green") - return credit + return background_config[2] diff --git a/video_creation/final_video.py b/video_creation/final_video.py index 1035483..ebdf4d0 100755 --- a/video_creation/final_video.py +++ b/video_creation/final_video.py @@ -49,10 +49,10 @@ def name_normalize(name: str) -> str: def make_final_video(number_of_clips: int, length: int, reddit_obj: dict, background_config: Tuple[str, str, str, Any]): """Gathers audio clips, gathers all screenshots, stitches them together and saves the final video to assets/temp Args: - number_of_clips (int): Index to end at when going through the screenshots + number_of_clips (int): Index to end at when going through the screenshots' length (int): Length of the video reddit_obj (dict): The reddit object that contains the posts to read. - background_config Tuple[str, str, str, Any]: The background config to use. + background_config (Tuple[str, str, str, Any]): The background config to use. """ print_step("Creating the final video 🎥") VideoFileClip.reW = lambda clip: clip.resize(width=W) @@ -103,7 +103,7 @@ def make_final_video(number_of_clips: int, length: int, reddit_obj: dict, backgr # .resize(width=W - 100) # .set_opacity(float(opacity)), # ) - # else: + # else: story mode stuff img_clip_pos = background_config[3] image_concat = concatenate_videoclips( image_clips).set_position(img_clip_pos) From c22856271c632d0778f43c0734972e4249374b4c Mon Sep 17 00:00:00 2001 From: Jason Date: Wed, 6 Jul 2022 23:32:30 -0400 Subject: [PATCH 072/126] fix: removed duplicate func call docs: added verq to the devs --- README.md | 6 ++---- main.py | 1 - 2 files changed, 2 insertions(+), 5 deletions(-) diff --git a/README.md b/README.md index 7f10287..af01afe 100644 --- a/README.md +++ b/README.md @@ -75,12 +75,10 @@ Please read our [contributing guidelines](CONTRIBUTING.md) for more detailed inf Elebumm (Lewis#6305) - https://github.com/elebumm (Founder) -Jason (JasonLovesDoggo#1904) - https://github.com/JasonLovesDoggo +Jason (JasonLovesDoggo#1904) - https://github.com/JasonLovesDoggo (Maintainer) CallumIO (c.#6837) - https://github.com/CallumIO -HarryDaDev (hrvyy#9677) - https://github.com/ImmaHarry +Verq (Verq#2338) - https://github.com/CordlessCoder LukaHietala (Pix.#0001) - https://github.com/LukaHietala - -Freebiell (Freebie#6429) - https://github.com/FreebieII diff --git a/main.py b/main.py index 622cd2e..cefb74d 100755 --- a/main.py +++ b/main.py @@ -36,7 +36,6 @@ def main(POST_ID=None): reddit_object = get_subreddit_threads(POST_ID) length, number_of_comments = save_text_to_mp3(reddit_object) length = math.ceil(length) - bg_config = get_background_config() download_screenshots_of_reddit_posts(reddit_object, number_of_comments) bg_config = get_background_config() download_background(bg_config) From 4c29214518d59668e706ce5dcd1f4da3c20a1a77 Mon Sep 17 00:00:00 2001 From: Jason Date: Wed, 6 Jul 2022 23:52:26 -0400 Subject: [PATCH 073/126] style/fix: fixed linter --- .github/workflows/lint.yml | 32 ++++++++++++++++++++++++++------ 1 file changed, 26 insertions(+), 6 deletions(-) diff --git a/.github/workflows/lint.yml b/.github/workflows/lint.yml index b04fb15..b473fbe 100644 --- a/.github/workflows/lint.yml +++ b/.github/workflows/lint.yml @@ -1,10 +1,30 @@ -name: Lint - -on: [push, pull_request] +# GitHub Action that uses Black to reformat the Python code in an incoming pull request. +# If all Python code in the pull request is compliant with Black then this Action does nothing. +# Othewrwise, Black is run and its changes are committed back to the incoming pull request. +# https://github.com/cclauss/autoblack +name: autoblack +on: [pull_request, workflow_dispatch] jobs: - lint: + build: runs-on: ubuntu-latest steps: - - uses: actions/checkout@v2 - - uses: psf/black@stable + - uses: actions/checkout@v1 + - name: Set up Python 3.7 + uses: actions/setup-python@v1 + with: + python-version: 3.9 + - name: Install Black + run: pip install black + - name: Run black --check . + run: black --check . + - name: If needed, commit black changes to the pull request + if: failure() + run: | + black . --line-length 101 + git config --global user.name 'autoblack' + git config --global user.email 'cclauss@users.noreply.github.com' + git remote set-url origin https://x-access-token:${{ secrets.GITHUB_TOKEN }}@github.com/$GITHUB_REPOSITORY + git checkout $GITHUB_HEAD_REF + git commit -am "fixup: Format Python code with Black" + git push From 0d63fb60c04d573ae806b9b9522bab5bd956fa67 Mon Sep 17 00:00:00 2001 From: Jason Date: Wed, 6 Jul 2022 23:54:32 -0400 Subject: [PATCH 074/126] style/fix: fixed linter --- .github/workflows/lint.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/lint.yml b/.github/workflows/lint.yml index b473fbe..a8fbb93 100644 --- a/.github/workflows/lint.yml +++ b/.github/workflows/lint.yml @@ -23,7 +23,7 @@ jobs: run: | black . --line-length 101 git config --global user.name 'autoblack' - git config --global user.email 'cclauss@users.noreply.github.com' + git config --global user.email 'JasonLovesDoggo@users.noreply.github.com' git remote set-url origin https://x-access-token:${{ secrets.GITHUB_TOKEN }}@github.com/$GITHUB_REPOSITORY git checkout $GITHUB_HEAD_REF git commit -am "fixup: Format Python code with Black" From fc414b57b7784448653de522de3de1f3cbe5eff3 Mon Sep 17 00:00:00 2001 From: Jason Date: Wed, 6 Jul 2022 23:57:11 -0400 Subject: [PATCH 075/126] style: reformatted --- TTS/TikTok.py | 12 +++++------- TTS/aws_polly.py | 4 +--- TTS/engine_wrapper.py | 12 +++--------- TTS/streamlabs_polly.py | 4 +--- main.py | 10 ++++++---- reddit/subreddit.py | 16 ++++------------ utils/cleanup.py | 4 +--- utils/console.py | 17 +++-------------- utils/subreddit.py | 4 +--- utils/videos.py | 4 +--- video_creation/background.py | 20 ++++++++++---------- video_creation/screenshot_downloader.py | 12 +++--------- video_creation/voices.py | 14 +++----------- 13 files changed, 42 insertions(+), 91 deletions(-) diff --git a/TTS/TikTok.py b/TTS/TikTok.py index 6a116d7..743118c 100644 --- a/TTS/TikTok.py +++ b/TTS/TikTok.py @@ -62,7 +62,9 @@ noneng = [ class TikTok: # TikTok Text-to-Speech Wrapper def __init__(self): - self.URI_BASE = "https://api16-normal-useast5.us.tiktokv.com/media/api/text/speech/invoke/?text_speaker=" + self.URI_BASE = ( + "https://api16-normal-useast5.us.tiktokv.com/media/api/text/speech/invoke/?text_speaker=" + ) self.max_chars = 300 self.voices = {"human": human, "nonhuman": nonhuman, "noneng": noneng} @@ -79,9 +81,7 @@ class TikTok: # TikTok Text-to-Speech Wrapper ) ) try: - r = requests.post( - f"{self.URI_BASE}{voice}&req_text={text}&speaker_map_type=0" - ) + r = requests.post(f"{self.URI_BASE}{voice}&req_text={text}&speaker_map_type=0") except requests.exceptions.SSLError: # https://stackoverflow.com/a/47475019/18516611 session = requests.Session() @@ -89,9 +89,7 @@ class TikTok: # TikTok Text-to-Speech Wrapper adapter = HTTPAdapter(max_retries=retry) session.mount("http://", adapter) session.mount("https://", adapter) - r = session.post( - f"{self.URI_BASE}{voice}&req_text={text}&speaker_map_type=0" - ) + r = session.post(f"{self.URI_BASE}{voice}&req_text={text}&speaker_map_type=0") # print(r.text) vstr = [r.json()["data"]["v_str"]][0] b64d = base64.b64decode(vstr) diff --git a/TTS/aws_polly.py b/TTS/aws_polly.py index 0ff0d74..13eaea1 100644 --- a/TTS/aws_polly.py +++ b/TTS/aws_polly.py @@ -39,9 +39,7 @@ class AWSPolly: return ValueError( f"Please set the TOML variable AWS_VOICE to a valid voice. options are: {voices}" ) - voice = str( - settings.config["settings"]["tts"]["aws_polly_voice"] - ).capitalize() + voice = str(settings.config["settings"]["tts"]["aws_polly_voice"]).capitalize() try: # Request speech synthesis response = polly.synthesize_speech( diff --git a/TTS/engine_wrapper.py b/TTS/engine_wrapper.py index e3cc04b..a171db7 100644 --- a/TTS/engine_wrapper.py +++ b/TTS/engine_wrapper.py @@ -63,9 +63,7 @@ class TTSEngine: self.call_tts("posttext", self.reddit_object["thread_post"]) idx = None - for idx, comment in track( - enumerate(self.reddit_object["comments"]), "Saving..." - ): + for idx, comment in track(enumerate(self.reddit_object["comments"]), "Saving..."): # ! Stop creating mp3 files if the length is greater than max length. if self.length > self.max_length: break @@ -81,9 +79,7 @@ class TTSEngine: split_files = [] split_text = [ x.group().strip() - for x in re.finditer( - rf" *((.{{0,{self.tts_module.max_chars}}})(\.|.$))", text - ) + for x in re.finditer(rf" *((.{{0,{self.tts_module.max_chars}}})(\.|.$))", text) ] idy = None @@ -106,9 +102,7 @@ class TTSEngine: # Path(f"{self.path}/{idx}-{i}.part.mp3").unlink() def call_tts(self, filename: str, text: str): - self.tts_module.run( - text=process_text(text), filepath=f"{self.path}/{filename}.mp3" - ) + self.tts_module.run(text=process_text(text), filepath=f"{self.path}/{filename}.mp3") # try: # self.length += MP3(f"{self.path}/{filename}.mp3").info.length # except (MutagenError, HeaderNotFoundError): diff --git a/TTS/streamlabs_polly.py b/TTS/streamlabs_polly.py index d4ddcb4..1f8a039 100644 --- a/TTS/streamlabs_polly.py +++ b/TTS/streamlabs_polly.py @@ -39,9 +39,7 @@ class StreamlabsPolly: return ValueError( f"Please set the config variable STREAMLABS_VOICE to a valid voice. options are: {voices}" ) - voice = str( - settings.config["settings"]["tts"]["streamlabs_polly_voice"] - ).capitalize() + voice = str(settings.config["settings"]["tts"]["streamlabs_polly_voice"]).capitalize() body = {"voice": voice, "text": text, "service": "polly"} response = requests.post(self.url, data=body) try: diff --git a/main.py b/main.py index cefb74d..8ce8725 100755 --- a/main.py +++ b/main.py @@ -8,7 +8,11 @@ from utils.console import print_markdown, print_step from utils import settings # from utils.checker import envUpdate -from video_creation.background import download_background, chop_background_video, get_background_config +from video_creation.background import ( + download_background, + chop_background_video, + get_background_config, +) from video_creation.final_video import make_final_video from video_creation.screenshot_downloader import download_screenshots_of_reddit_posts from video_creation.voices import save_text_to_mp3 @@ -60,9 +64,7 @@ if __name__ == "__main__": run_many(config["settings"]["times_to_run"]) elif len(config["reddit"]["thread"]["post_id"].split("+")) > 1: - for index, post_id in enumerate( - config["reddit"]["thread"]["post_id"].split("+") - ): + for index, post_id in enumerate(config["reddit"]["thread"]["post_id"].split("+")): index += 1 print_step( f'on the {index}{("st" if index%10 == 1 else ("nd" if index%10 == 2 else ("rd" if index%10 == 3 else "th")))} post of {len(config["reddit"]["thread"]["post_id"].split("+"))}' diff --git a/reddit/subreddit.py b/reddit/subreddit.py index 7583653..8365d04 100644 --- a/reddit/subreddit.py +++ b/reddit/subreddit.py @@ -18,9 +18,7 @@ def get_subreddit_threads(POST_ID: str): content = {} if settings.config["reddit"]["creds"]["2fa"] == True: - print( - "\nEnter your two-factor authentication code from your authenticator app.\n" - ) + print("\nEnter your two-factor authentication code from your authenticator app.\n") code = input("> ") print() pw = settings.config["reddit"]["creds"]["password"] @@ -46,9 +44,7 @@ def get_subreddit_threads(POST_ID: str): ]: # note to user. you can have multiple subreddits via reddit.subreddit("redditdev+learnpython") try: subreddit = reddit.subreddit( - re.sub( - r"r\/", "", input("What subreddit would you like to pull from? ") - ) + re.sub(r"r\/", "", input("What subreddit would you like to pull from? ")) # removes the r/ from the input ) except ValueError: @@ -58,9 +54,7 @@ def get_subreddit_threads(POST_ID: str): sub = settings.config["reddit"]["thread"]["subreddit"] print_substep(f"Using subreddit: r/{sub} from TOML config") subreddit_choice = sub - if subreddit_choice.casefold().startswith( - "r/" - ): # removes the r/ from the input + if subreddit_choice.casefold().startswith("r/"): # removes the r/ from the input subreddit_choice = subreddit_choice[2:] subreddit = reddit.subreddit( subreddit_choice @@ -72,9 +66,7 @@ def get_subreddit_threads(POST_ID: str): settings.config["reddit"]["thread"]["post_id"] and len(settings.config["reddit"]["thread"]["post_id"].split("+")) == 1 ): - submission = reddit.submission( - id=settings.config["reddit"]["thread"]["post_id"] - ) + submission = reddit.submission(id=settings.config["reddit"]["thread"]["post_id"]) else: threads = subreddit.hot(limit=25) diff --git a/utils/cleanup.py b/utils/cleanup.py index 44629a9..ef4fc44 100644 --- a/utils/cleanup.py +++ b/utils/cleanup.py @@ -10,9 +10,7 @@ def cleanup() -> int: """ if exists("./assets/temp"): count = 0 - files = [ - f for f in os.listdir(".") if f.endswith(".mp4") and "temp" in f.lower() - ] + files = [f for f in os.listdir(".") if f.endswith(".mp4") and "temp" in f.lower()] count += len(files) for f in files: os.remove(f) diff --git a/utils/console.py b/utils/console.py index 1ffa11c..6f99a41 100644 --- a/utils/console.py +++ b/utils/console.py @@ -49,10 +49,7 @@ def handle_input( optional=False, ): if optional: - console.print( - message - + "\n[green]This is an optional value. Do you want to skip it? (y/n)" - ) + console.print(message + "\n[green]This is an optional value. Do you want to skip it? (y/n)") if input().casefold().startswith("y"): return default if default is not NotImplemented else "" if default is not NotImplemented: @@ -86,11 +83,7 @@ def handle_input( console.print("[red]" + err_message) continue elif match != "" and re.match(match, user_input) is None: - console.print( - "[red]" - + err_message - + "\nAre you absolutely sure it's correct?(y/n)" - ) + console.print("[red]" + err_message + "\nAre you absolutely sure it's correct?(y/n)") if input().casefold().startswith("y"): break continue @@ -123,9 +116,5 @@ def handle_input( if user_input in options: return user_input console.print( - "[red bold]" - + err_message - + "\nValid options are: " - + ", ".join(map(str, options)) - + "." + "[red bold]" + err_message + "\nValid options are: " + ", ".join(map(str, options)) + "." ) diff --git a/utils/subreddit.py b/utils/subreddit.py index 7647454..140f4d3 100644 --- a/utils/subreddit.py +++ b/utils/subreddit.py @@ -15,9 +15,7 @@ def get_subreddit_undone(submissions: list, subreddit): """ # recursively checks if the top submission in the list was already done. - with open( - "./video_creation/data/videos.json", "r", encoding="utf-8" - ) as done_vids_raw: + with open("./video_creation/data/videos.json", "r", encoding="utf-8") as done_vids_raw: done_videos = json.load(done_vids_raw) for submission in submissions: if already_done(done_videos, submission): diff --git a/utils/videos.py b/utils/videos.py index 51143c3..d050ab8 100755 --- a/utils/videos.py +++ b/utils/videos.py @@ -20,9 +20,7 @@ def check_done( Returns: Submission|None: Reddit object in args """ - with open( - "./video_creation/data/videos.json", "r", encoding="utf-8" - ) as done_vids_raw: + with open("./video_creation/data/videos.json", "r", encoding="utf-8") as done_vids_raw: done_videos = json.load(done_vids_raw) for video in done_videos: if video["id"] == str(redditobj): diff --git a/video_creation/background.py b/video_creation/background.py index d9dc056..7ef4321 100644 --- a/video_creation/background.py +++ b/video_creation/background.py @@ -24,28 +24,29 @@ background_options = { "https://www.youtube.com/watch?v=vw5L4xCPy9Q", "bike-parkour-gta.mp4", "Achy Gaming", - lambda t: ('center', 480 + t) + lambda t: ("center", 480 + t), ), "rocket-league": ( # Rocket League "https://www.youtube.com/watch?v=2X9QGY__0II", "rocket_league.mp4", "Orbital Gameplay", - "top" + "top", ), "minecraft": ( # Minecraft parkour "https://www.youtube.com/watch?v=n_Dv4JMiwK8", "parkour.mp4", "bbswitzer", - "center" + "center", ), "gta": ( # GTA Stunt Race "https://www.youtube.com/watch?v=qGa9kWREOnE", "gta-stunt-race.mp4", "Achy Gaming", - lambda t: ('center', 480 + t) - ) + lambda t: ("center", 480 + t), + ), } + def get_start_and_end_times(video_length: int, length_of_clip: int) -> Tuple[int, int]: """Generates a random interval of time to be used as the background of the video. @@ -59,10 +60,11 @@ def get_start_and_end_times(video_length: int, length_of_clip: int) -> Tuple[int random_time = randrange(180, int(length_of_clip) - int(video_length)) return random_time, random_time + video_length + def get_background_config(): """Fetch the background/s configuration""" try: - choice = str(settings.config['settings']['background_choice']).casefold() + choice = str(settings.config["settings"]["background_choice"]).casefold() except AttributeError: print_substep("No background selected. Picking random background'") choice = None @@ -90,8 +92,7 @@ def download_background(background_config: Tuple[str, str, str, Any]): YouTube(uri, on_progress_callback=on_progress).streams.filter(res="1080p").first().download( "assets/backgrounds", filename=f"{credit}-{filename}" ) - print_substep("Background videos downloaded successfully! 🎉", - style="bold green") + print_substep("Background videos downloaded successfully! 🎉", style="bold green") def chop_background_video(background_config: Tuple[str, str, str, Any], video_length: int): @@ -107,8 +108,7 @@ def chop_background_video(background_config: Tuple[str, str, str, Any], video_le background = VideoFileClip(f"assets/backgrounds/{choice}") - start_time, end_time = get_start_and_end_times( - video_length, background.duration) + start_time, end_time = get_start_and_end_times(video_length, background.duration) try: ffmpeg_extract_subclip( f"assets/backgrounds/{choice}", diff --git a/video_creation/screenshot_downloader.py b/video_creation/screenshot_downloader.py index 9dbb3d8..6fb9ef4 100644 --- a/video_creation/screenshot_downloader.py +++ b/video_creation/screenshot_downloader.py @@ -35,13 +35,9 @@ def download_screenshots_of_reddit_posts(reddit_object: dict, screenshot_num: in context = browser.new_context() if settings.config["settings"]["theme"] == "dark": - cookie_file = open( - "./video_creation/data/cookie-dark-mode.json", encoding="utf-8" - ) + cookie_file = open("./video_creation/data/cookie-dark-mode.json", encoding="utf-8") else: - cookie_file = open( - "./video_creation/data/cookie-light-mode.json", encoding="utf-8" - ) + cookie_file = open("./video_creation/data/cookie-light-mode.json", encoding="utf-8") cookies = json.load(cookie_file) context.add_cookies(cookies) # load preference cookies # Get the thread screenshot @@ -73,9 +69,7 @@ def download_screenshots_of_reddit_posts(reddit_object: dict, screenshot_num: in else: print_substep("Skipping translation...") - page.locator('[data-test-id="post-content"]').screenshot( - path="assets/temp/png/title.png" - ) + page.locator('[data-test-id="post-content"]').screenshot(path="assets/temp/png/title.png") if storymode: page.locator('[data-click-id="text"]').screenshot( diff --git a/video_creation/voices.py b/video_creation/voices.py index e6e1045..ffc0898 100644 --- a/video_creation/voices.py +++ b/video_creation/voices.py @@ -35,9 +35,7 @@ def save_text_to_mp3(reddit_obj) -> Tuple[int, int]: voice = settings.config["settings"]["tts"]["choice"] if voice.casefold() in map(lambda _: _.casefold(), TTSProviders): - text_to_mp3 = TTSEngine( - get_case_insensitive_key_value(TTSProviders, voice), reddit_obj - ) + text_to_mp3 = TTSEngine(get_case_insensitive_key_value(TTSProviders, voice), reddit_obj) else: while True: print_step("Please choose one of the following TTS providers: ") @@ -46,19 +44,13 @@ def save_text_to_mp3(reddit_obj) -> Tuple[int, int]: if choice.casefold() in map(lambda _: _.casefold(), TTSProviders): break print("Unknown Choice") - text_to_mp3 = TTSEngine( - get_case_insensitive_key_value(TTSProviders, choice), reddit_obj - ) + text_to_mp3 = TTSEngine(get_case_insensitive_key_value(TTSProviders, choice), reddit_obj) return text_to_mp3.run() def get_case_insensitive_key_value(input_dict, key): return next( - ( - value - for dict_key, value in input_dict.items() - if dict_key.lower() == key.lower() - ), + (value for dict_key, value in input_dict.items() if dict_key.lower() == key.lower()), None, ) From 550b6c6aae7c7565217eedc5223aa06d07226514 Mon Sep 17 00:00:00 2001 From: Jason Date: Wed, 6 Jul 2022 23:59:50 -0400 Subject: [PATCH 076/126] fix: added a check if videos.json exists. --- utils/subreddit.py | 8 ++++++-- 1 file changed, 6 insertions(+), 2 deletions(-) diff --git a/utils/subreddit.py b/utils/subreddit.py index 140f4d3..9c6ef31 100644 --- a/utils/subreddit.py +++ b/utils/subreddit.py @@ -1,4 +1,6 @@ import json +from os.path import exists + from utils import settings from utils.console import print_substep @@ -14,7 +16,9 @@ def get_subreddit_undone(submissions: list, subreddit): Any: The submission that has not been done """ # recursively checks if the top submission in the list was already done. - + if not exists('./video_creation/data/videos.json'): + with open('./video_creation/data/videos.json', 'w+') as f: + json.dump([], f) with open("./video_creation/data/videos.json", "r", encoding="utf-8") as done_vids_raw: done_videos = json.load(done_vids_raw) for submission in submissions: @@ -22,7 +26,7 @@ def get_subreddit_undone(submissions: list, subreddit): continue if submission.over_18: try: - if settings.config["settings"]["allow_nsfw"] == False: + if not settings.config["settings"]["allow_nsfw"]: print_substep("NSFW Post Detected. Skipping...") continue except AttributeError: From 68a643ebec6f702f38706c83ffe43dc96456ba68 Mon Sep 17 00:00:00 2001 From: Jason Date: Thu, 7 Jul 2022 00:15:01 -0400 Subject: [PATCH 077/126] Update lint.yml --- .github/workflows/lint.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/lint.yml b/.github/workflows/lint.yml index a8fbb93..0ca5c5d 100644 --- a/.github/workflows/lint.yml +++ b/.github/workflows/lint.yml @@ -23,7 +23,7 @@ jobs: run: | black . --line-length 101 git config --global user.name 'autoblack' - git config --global user.email 'JasonLovesDoggo@users.noreply.github.com' + git config --global user.email 'jasoncameron.all@gmail.com' git remote set-url origin https://x-access-token:${{ secrets.GITHUB_TOKEN }}@github.com/$GITHUB_REPOSITORY git checkout $GITHUB_HEAD_REF git commit -am "fixup: Format Python code with Black" From 3e816b9f8aca1269bddf1e1d6bece773a65cd723 Mon Sep 17 00:00:00 2001 From: Jason Date: Thu, 7 Jul 2022 00:15:46 -0400 Subject: [PATCH 078/126] style: refactor --- utils/settings.py | 35 ++++++++--------------------------- 1 file changed, 8 insertions(+), 27 deletions(-) diff --git a/utils/settings.py b/utils/settings.py index 7c2b05d..a36f63e 100755 --- a/utils/settings.py +++ b/utils/settings.py @@ -11,8 +11,9 @@ from utils.console import handle_input console = Console() config = dict # autocomplete + def crawl(obj: dict, func=lambda x, y: print(x, y, end="\n"), path=None): - if path is None: # path Default argument value is mutable + if path is None: # path Default argument value is mutable path = [] for key in obj.keys(): if type(obj[key]) is dict: @@ -53,11 +54,7 @@ def check(value, checks, name): and not hasattr(value, "__iter__") and ( ("nmin" in checks and checks["nmin"] is not None and value < checks["nmin"]) - or ( - "nmax" in checks - and checks["nmax"] is not None - and value > checks["nmax"] - ) + or ("nmax" in checks and checks["nmax"] is not None and value > checks["nmax"]) ) ): incorrect = True @@ -65,16 +62,8 @@ def check(value, checks, name): not incorrect and hasattr(value, "__iter__") and ( - ( - "nmin" in checks - and checks["nmin"] is not None - and len(value) < checks["nmin"] - ) - or ( - "nmax" in checks - and checks["nmax"] is not None - and len(value) > checks["nmax"] - ) + ("nmin" in checks and checks["nmin"] is not None and len(value) < checks["nmin"]) + or ("nmax" in checks and checks["nmax"] is not None and len(value) > checks["nmax"]) ) ): incorrect = True @@ -82,15 +71,9 @@ def check(value, checks, name): if incorrect: value = handle_input( message=( - ( - ("[blue]Example: " + str(checks["example"]) + "\n") - if "example" in checks - else "" - ) + (("[blue]Example: " + str(checks["example"]) + "\n") if "example" in checks else "") + "[red]" - + ("Non-optional ", "Optional ")[ - "optional" in checks and checks["optional"] is True - ] + + ("Non-optional ", "Optional ")["optional" in checks and checks["optional"] is True] ) + "[#C0CAF5 bold]" + str(name) @@ -131,9 +114,7 @@ def check_toml(template_file, config_file) -> Tuple[bool, Dict]: try: template = toml.load(template_file) except Exception as error: - console.print( - f"[red bold]Encountered error when trying to to load {template_file}: {error}" - ) + console.print(f"[red bold]Encountered error when trying to to load {template_file}: {error}") return False try: config = toml.load(config_file) From e44718ffdb634c174b77b349fb55a15c9b5ab3d7 Mon Sep 17 00:00:00 2001 From: Jason Date: Thu, 7 Jul 2022 00:28:20 -0400 Subject: [PATCH 079/126] feat: background audio implementation closes #11 (note: not my finest work, I wrote this sleep-deprived) --- .config.template.toml | 6 +++-- video_creation/final_video.py | 47 ++++++++++++++++++++++++----------- 2 files changed, 36 insertions(+), 17 deletions(-) diff --git a/.config.template.toml b/.config.template.toml index 8775357..4da51f7 100644 --- a/.config.template.toml +++ b/.config.template.toml @@ -28,9 +28,11 @@ times_to_run = { optional = false, default = 1, example = 2, explanation = "used opacity = { optional = false, default = 0.9, example = 0.8, explanation = "Sets the opacity of the comments when overlayed over the background", type = "float", nmin = 0, nmax = 1, oob_error = "The opacity HAS to be between 0 and 1", input_error = "The opacity HAS to be a decimal number between 0 and 1" } storymode = { optional = true, type = "bool", default = false, example = false, options = [true, false, -] } +], explanation = "not yet implemented" } background_choice = { optional = true, default = "minecraft", example = "minecraft", options = ["minecraft", "gta", "rocket-league", "motor-gta"], explanation = "Sets the background for the video" } - +background_audio = { optional = true, type = "bool", default = false, example = false, options = [true, + false, +], explaination="Sets a audio to play in the background (put a background.mp3 file in the assets/backgrounds directory for it to be used.)" } [settings.tts] choice = { optional = false, default = "", options = ["streamlabspolly", "tiktok", "googletranslate", "awspolly", ], example = "streamlabspolly", explanation = "The backend used for TTS generation. This can be left blank and you will be prompted to choose at runtime." } aws_polly_voice = { optional = false, default = "Matthew", example = "Matthew", explanation = "The voice used for AWS Polly" } diff --git a/video_creation/final_video.py b/video_creation/final_video.py index ebdf4d0..eed072a 100755 --- a/video_creation/final_video.py +++ b/video_creation/final_video.py @@ -4,6 +4,7 @@ import os import re from os.path import exists from typing import Dict, Tuple, Any + import translators as ts from moviepy.editor import ( @@ -15,7 +16,7 @@ from moviepy.editor import ( CompositeAudioClip, CompositeVideoClip, ) -from moviepy.video.io import ffmpeg_tools +from moviepy.video.io.ffmpeg_tools import ffmpeg_merge_video_audio, ffmpeg_extract_subclip from rich.console import Console from utils.cleanup import cleanup @@ -46,7 +47,10 @@ def name_normalize(name: str) -> str: else: return name -def make_final_video(number_of_clips: int, length: int, reddit_obj: dict, background_config: Tuple[str, str, str, Any]): + +def make_final_video( + number_of_clips: int, length: int, reddit_obj: dict, background_config: Tuple[str, str, str, Any] +): """Gathers audio clips, gathers all screenshots, stitches them together and saves the final video to assets/temp Args: number_of_clips (int): Index to end at when going through the screenshots' @@ -66,9 +70,7 @@ def make_final_video(number_of_clips: int, length: int, reddit_obj: dict, backgr ) # Gather all audio clips - audio_clips = [ - AudioFileClip(f"assets/temp/mp3/{i}.mp3") for i in range(number_of_clips) - ] + audio_clips = [AudioFileClip(f"assets/temp/mp3/{i}.mp3") for i in range(number_of_clips)] audio_clips.insert(0, AudioFileClip("assets/temp/mp3/title.mp3")) audio_concat = concatenate_audioclips(audio_clips) audio_composite = CompositeAudioClip([audio_concat]) @@ -105,8 +107,7 @@ def make_final_video(number_of_clips: int, length: int, reddit_obj: dict, backgr # ) # else: story mode stuff img_clip_pos = background_config[3] - image_concat = concatenate_videoclips( - image_clips).set_position(img_clip_pos) + image_concat = concatenate_videoclips(image_clips).set_position(img_clip_pos) image_concat.audio = audio_composite final = CompositeVideoClip([background_clip, image_concat]) title = re.sub(r"[^\w\s-]", "", reddit_obj["thread_title"]) @@ -129,14 +130,30 @@ def make_final_video(number_of_clips: int, length: int, reddit_obj: dict, backgr verbose=False, threads=multiprocessing.cpu_count(), ) - ffmpeg_tools.ffmpeg_extract_subclip( - "assets/temp/temp.mp4", - 0, - final.duration, - targetname=f"results/{subreddit}/{filename}", - ) - # os.remove("assets/temp/temp.mp4") - + if settings.config["settings"]["background_audio"]: # background.mp3 + if not exists(f"assets/mp3/background.mp3"): + print_substep("optional background audio file didn't so skipping.") + ffmpeg_extract_subclip( + "assets/temp/temp.mp4", + 0, + final.duration, + targetname=f"results/{subreddit}/{filename}", + ) + else: + ffmpeg_merge_video_audio("assets/temp/temp.mp4", "assets/backgrounds/background.mp3", "assets/temp/temp_audio.mp4") + ffmpeg_extract_subclip( + "assets/temp/temp_audio.mp4", + 0, + final.duration, + targetname=f"results/{subreddit}/{filename}", + ) + else: + ffmpeg_extract_subclip( + "assets/temp/temp.mp4", + 0, + final.duration, + targetname=f"results/{subreddit}/{filename}", + ) print_step("Removing temporary files 🗑") cleanups = cleanup() print_substep(f"Removed {cleanups} temporary files 🗑") From dd991e472311ae267d129cf67d8c5ea23ae24351 Mon Sep 17 00:00:00 2001 From: Jason Date: Thu, 7 Jul 2022 00:15:01 -0400 Subject: [PATCH 080/126] Update lint.yml --- .github/workflows/lint.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/lint.yml b/.github/workflows/lint.yml index a8fbb93..0ca5c5d 100644 --- a/.github/workflows/lint.yml +++ b/.github/workflows/lint.yml @@ -23,7 +23,7 @@ jobs: run: | black . --line-length 101 git config --global user.name 'autoblack' - git config --global user.email 'JasonLovesDoggo@users.noreply.github.com' + git config --global user.email 'jasoncameron.all@gmail.com' git remote set-url origin https://x-access-token:${{ secrets.GITHUB_TOKEN }}@github.com/$GITHUB_REPOSITORY git checkout $GITHUB_HEAD_REF git commit -am "fixup: Format Python code with Black" From 40bfcf468e63cc6a2da0f73a59fb6c5f5588ca70 Mon Sep 17 00:00:00 2001 From: autoblack Date: Thu, 7 Jul 2022 04:38:26 +0000 Subject: [PATCH 081/126] fixup: Format Python code with Black --- utils/settings.py | 35 ++++++++--------------------------- video_creation/final_video.py | 12 ++++++------ 2 files changed, 14 insertions(+), 33 deletions(-) diff --git a/utils/settings.py b/utils/settings.py index 7c2b05d..a36f63e 100755 --- a/utils/settings.py +++ b/utils/settings.py @@ -11,8 +11,9 @@ from utils.console import handle_input console = Console() config = dict # autocomplete + def crawl(obj: dict, func=lambda x, y: print(x, y, end="\n"), path=None): - if path is None: # path Default argument value is mutable + if path is None: # path Default argument value is mutable path = [] for key in obj.keys(): if type(obj[key]) is dict: @@ -53,11 +54,7 @@ def check(value, checks, name): and not hasattr(value, "__iter__") and ( ("nmin" in checks and checks["nmin"] is not None and value < checks["nmin"]) - or ( - "nmax" in checks - and checks["nmax"] is not None - and value > checks["nmax"] - ) + or ("nmax" in checks and checks["nmax"] is not None and value > checks["nmax"]) ) ): incorrect = True @@ -65,16 +62,8 @@ def check(value, checks, name): not incorrect and hasattr(value, "__iter__") and ( - ( - "nmin" in checks - and checks["nmin"] is not None - and len(value) < checks["nmin"] - ) - or ( - "nmax" in checks - and checks["nmax"] is not None - and len(value) > checks["nmax"] - ) + ("nmin" in checks and checks["nmin"] is not None and len(value) < checks["nmin"]) + or ("nmax" in checks and checks["nmax"] is not None and len(value) > checks["nmax"]) ) ): incorrect = True @@ -82,15 +71,9 @@ def check(value, checks, name): if incorrect: value = handle_input( message=( - ( - ("[blue]Example: " + str(checks["example"]) + "\n") - if "example" in checks - else "" - ) + (("[blue]Example: " + str(checks["example"]) + "\n") if "example" in checks else "") + "[red]" - + ("Non-optional ", "Optional ")[ - "optional" in checks and checks["optional"] is True - ] + + ("Non-optional ", "Optional ")["optional" in checks and checks["optional"] is True] ) + "[#C0CAF5 bold]" + str(name) @@ -131,9 +114,7 @@ def check_toml(template_file, config_file) -> Tuple[bool, Dict]: try: template = toml.load(template_file) except Exception as error: - console.print( - f"[red bold]Encountered error when trying to to load {template_file}: {error}" - ) + console.print(f"[red bold]Encountered error when trying to to load {template_file}: {error}") return False try: config = toml.load(config_file) diff --git a/video_creation/final_video.py b/video_creation/final_video.py index ebdf4d0..d08e4cb 100755 --- a/video_creation/final_video.py +++ b/video_creation/final_video.py @@ -46,7 +46,10 @@ def name_normalize(name: str) -> str: else: return name -def make_final_video(number_of_clips: int, length: int, reddit_obj: dict, background_config: Tuple[str, str, str, Any]): + +def make_final_video( + number_of_clips: int, length: int, reddit_obj: dict, background_config: Tuple[str, str, str, Any] +): """Gathers audio clips, gathers all screenshots, stitches them together and saves the final video to assets/temp Args: number_of_clips (int): Index to end at when going through the screenshots' @@ -66,9 +69,7 @@ def make_final_video(number_of_clips: int, length: int, reddit_obj: dict, backgr ) # Gather all audio clips - audio_clips = [ - AudioFileClip(f"assets/temp/mp3/{i}.mp3") for i in range(number_of_clips) - ] + audio_clips = [AudioFileClip(f"assets/temp/mp3/{i}.mp3") for i in range(number_of_clips)] audio_clips.insert(0, AudioFileClip("assets/temp/mp3/title.mp3")) audio_concat = concatenate_audioclips(audio_clips) audio_composite = CompositeAudioClip([audio_concat]) @@ -105,8 +106,7 @@ def make_final_video(number_of_clips: int, length: int, reddit_obj: dict, backgr # ) # else: story mode stuff img_clip_pos = background_config[3] - image_concat = concatenate_videoclips( - image_clips).set_position(img_clip_pos) + image_concat = concatenate_videoclips(image_clips).set_position(img_clip_pos) image_concat.audio = audio_composite final = CompositeVideoClip([background_clip, image_concat]) title = re.sub(r"[^\w\s-]", "", reddit_obj["thread_title"]) From 5455b7ef25f26bcd9ccdf116d4e6573917306294 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Thu, 7 Jul 2022 10:47:20 +0000 Subject: [PATCH 082/126] Bump boto3 from 1.24.23 to 1.24.24 Bumps [boto3](https://github.com/boto/boto3) from 1.24.23 to 1.24.24. - [Release notes](https://github.com/boto/boto3/releases) - [Changelog](https://github.com/boto/boto3/blob/develop/CHANGELOG.rst) - [Commits](https://github.com/boto/boto3/compare/1.24.23...1.24.24) --- updated-dependencies: - dependency-name: boto3 dependency-type: direct:production update-type: version-update:semver-patch ... Signed-off-by: dependabot[bot] --- requirements.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/requirements.txt b/requirements.txt index 38df393..85d3d47 100644 --- a/requirements.txt +++ b/requirements.txt @@ -1,4 +1,4 @@ -boto3==1.24.23 +boto3==1.24.24 botocore==1.27.23 gTTS==2.2.4 moviepy==1.0.3 From 303efb8052650a251717eddcbe386b3622084ca2 Mon Sep 17 00:00:00 2001 From: micziz <99438936+micziz@users.noreply.github.com> Date: Thu, 7 Jul 2022 13:20:41 +0200 Subject: [PATCH 083/126] Update requirements.txt --- requirements.txt | 1 - 1 file changed, 1 deletion(-) diff --git a/requirements.txt b/requirements.txt index cc99eda..38df393 100644 --- a/requirements.txt +++ b/requirements.txt @@ -10,4 +10,3 @@ pytube==12.1.0 requests==2.28.1 rich==12.4.4 translators==5.3.1 -sox==1.4.1 From 532e49276a7331e0db78636109e62f349ec01d98 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Muhammed=20Mustafa=20Ak=C5=9Fam?= Date: Thu, 7 Jul 2022 14:36:23 +0300 Subject: [PATCH 084/126] Adding subreddit to videos.json Adding subreddit data to videos.json --- utils/videos.py | 4 +++- video_creation/final_video.py | 2 +- 2 files changed, 4 insertions(+), 2 deletions(-) diff --git a/utils/videos.py b/utils/videos.py index d050ab8..4a91e8c 100755 --- a/utils/videos.py +++ b/utils/videos.py @@ -34,11 +34,12 @@ def check_done( return redditobj -def save_data(filename: str, reddit_title: str, reddit_id: str, credit: str): +def save_data(subreddit: str, filename: str, reddit_title: str, reddit_id: str, credit: str): """Saves the videos that have already been generated to a JSON file in video_creation/data/videos.json Args: filename (str): The finished video title name + @param subreddit: @param filename: @param reddit_id: @param reddit_title: @@ -48,6 +49,7 @@ def save_data(filename: str, reddit_title: str, reddit_id: str, credit: str): if reddit_id in [video["id"] for video in done_vids]: return # video already done but was specified to continue anyway in the config file payload = { + "subreddit": subreddit, "id": reddit_id, "time": str(int(time.time())), "background_credit": credit, diff --git a/video_creation/final_video.py b/video_creation/final_video.py index d08e4cb..1dad789 100755 --- a/video_creation/final_video.py +++ b/video_creation/final_video.py @@ -115,7 +115,7 @@ def make_final_video( filename = f"{name_normalize(title)}.mp4" subreddit = settings.config["reddit"]["thread"]["subreddit"] - save_data(filename, title, idx, background_config[2]) + save_data(subreddit, filename, title, idx, background_config[2]) if not exists(f"./results/{subreddit}"): print_substep("The results folder didn't exist so I made it") From bd529add7d963f2caf5bdcf45ace9b7425f54fa6 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Thu, 7 Jul 2022 17:28:13 +0000 Subject: [PATCH 085/126] Bump botocore from 1.27.23 to 1.27.24 Bumps [botocore](https://github.com/boto/botocore) from 1.27.23 to 1.27.24. - [Release notes](https://github.com/boto/botocore/releases) - [Changelog](https://github.com/boto/botocore/blob/develop/CHANGELOG.rst) - [Commits](https://github.com/boto/botocore/compare/1.27.23...1.27.24) --- updated-dependencies: - dependency-name: botocore dependency-type: direct:production update-type: version-update:semver-patch ... Signed-off-by: dependabot[bot] --- requirements.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/requirements.txt b/requirements.txt index 85d3d47..40c7632 100644 --- a/requirements.txt +++ b/requirements.txt @@ -1,5 +1,5 @@ boto3==1.24.24 -botocore==1.27.23 +botocore==1.27.24 gTTS==2.2.4 moviepy==1.0.3 mutagen==1.45.1 From b51ba9a002dd7fdfbbd3a58862f9168921e61438 Mon Sep 17 00:00:00 2001 From: Callum Leslie Date: Thu, 7 Jul 2022 22:43:30 +0100 Subject: [PATCH 086/126] chore: lint on pr, lint+fix on push to master --- .github/workflows/autoblack.yml | 32 ++++++++++++++++++++++++++++++++ .github/workflows/lint.yml | 32 ++++++-------------------------- 2 files changed, 38 insertions(+), 26 deletions(-) create mode 100644 .github/workflows/autoblack.yml diff --git a/.github/workflows/autoblack.yml b/.github/workflows/autoblack.yml new file mode 100644 index 0000000..ba9cc36 --- /dev/null +++ b/.github/workflows/autoblack.yml @@ -0,0 +1,32 @@ +# GitHub Action that uses Black to reformat the Python code in an incoming pull request. +# If all Python code in the pull request is compliant with Black then this Action does nothing. +# Othewrwise, Black is run and its changes are committed back to the incoming pull request. +# https://github.com/cclauss/autoblack + +name: autoblack +on: + push: + branches: ["master"] +jobs: + build: + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v1 + - name: Set up Python 3.7 + uses: actions/setup-python@v1 + with: + python-version: 3.9 + - name: Install Black + run: pip install black + - name: Run black --check . + run: black --check . + - name: If needed, commit black changes to the pull request + if: failure() + run: | + black . --line-length 101 + git config --global user.name 'autoblack' + git config --global user.email 'jasoncameron.all@gmail.com' + git remote set-url origin https://x-access-token:${{ secrets.GITHUB_TOKEN }}@github.com/$GITHUB_REPOSITORY + git checkout $GITHUB_HEAD_REF + git commit -am "fixup: Format Python code with Black" + git push diff --git a/.github/workflows/lint.yml b/.github/workflows/lint.yml index 0ca5c5d..75ad759 100644 --- a/.github/workflows/lint.yml +++ b/.github/workflows/lint.yml @@ -1,30 +1,10 @@ -# GitHub Action that uses Black to reformat the Python code in an incoming pull request. -# If all Python code in the pull request is compliant with Black then this Action does nothing. -# Othewrwise, Black is run and its changes are committed back to the incoming pull request. -# https://github.com/cclauss/autoblack +name: Lint + +on: [pull_request] -name: autoblack -on: [pull_request, workflow_dispatch] jobs: - build: + lint: runs-on: ubuntu-latest steps: - - uses: actions/checkout@v1 - - name: Set up Python 3.7 - uses: actions/setup-python@v1 - with: - python-version: 3.9 - - name: Install Black - run: pip install black - - name: Run black --check . - run: black --check . - - name: If needed, commit black changes to the pull request - if: failure() - run: | - black . --line-length 101 - git config --global user.name 'autoblack' - git config --global user.email 'jasoncameron.all@gmail.com' - git remote set-url origin https://x-access-token:${{ secrets.GITHUB_TOKEN }}@github.com/$GITHUB_REPOSITORY - git checkout $GITHUB_HEAD_REF - git commit -am "fixup: Format Python code with Black" - git push + - uses: actions/checkout@v2 + - uses: psf/black@stable From d6be83ef80fec7e9ecb771c05b8c76ae6771231a Mon Sep 17 00:00:00 2001 From: Callum Leslie Date: Thu, 7 Jul 2022 21:52:45 +0000 Subject: [PATCH 087/126] fix: match lint line length to autoblack --- .github/workflows/lint.yml | 2 ++ 1 file changed, 2 insertions(+) diff --git a/.github/workflows/lint.yml b/.github/workflows/lint.yml index 75ad759..f95531f 100644 --- a/.github/workflows/lint.yml +++ b/.github/workflows/lint.yml @@ -8,3 +8,5 @@ jobs: steps: - uses: actions/checkout@v2 - uses: psf/black@stable + with: + options: "--line-length 101" From 83ba6346d09e3774bcfbceb73b2028b1288d8e9f Mon Sep 17 00:00:00 2001 From: Jason Date: Thu, 7 Jul 2022 20:57:27 -0400 Subject: [PATCH 088/126] fixes random background being blacked --- .config.template.toml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.config.template.toml b/.config.template.toml index 4da51f7..ddfa293 100644 --- a/.config.template.toml +++ b/.config.template.toml @@ -29,7 +29,7 @@ opacity = { optional = false, default = 0.9, example = 0.8, explanation = "Sets storymode = { optional = true, type = "bool", default = false, example = false, options = [true, false, ], explanation = "not yet implemented" } -background_choice = { optional = true, default = "minecraft", example = "minecraft", options = ["minecraft", "gta", "rocket-league", "motor-gta"], explanation = "Sets the background for the video" } +background_choice = { optional = true, default = "minecraft", example = "minecraft", options = ["minecraft", "gta", "rocket-league", "motor-gta", ""], explanation = "Sets the background for the video" } background_audio = { optional = true, type = "bool", default = false, example = false, options = [true, false, ], explaination="Sets a audio to play in the background (put a background.mp3 file in the assets/backgrounds directory for it to be used.)" } From e6169ee905464d5b4a2cf1ec74db0e9fea189133 Mon Sep 17 00:00:00 2001 From: Callum Leslie Date: Fri, 8 Jul 2022 02:18:08 +0100 Subject: [PATCH 089/126] fix: skip posts with no comments --- reddit/subreddit.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/reddit/subreddit.py b/reddit/subreddit.py index 8365d04..b64a52a 100644 --- a/reddit/subreddit.py +++ b/reddit/subreddit.py @@ -72,7 +72,7 @@ def get_subreddit_threads(POST_ID: str): threads = subreddit.hot(limit=25) submission = get_subreddit_undone(threads, subreddit) submission = check_done(submission) # double-checking - if submission is None: + if submission is None or not submission.num_comments: return get_subreddit_threads(POST_ID) # submission already done. rerun upvotes = submission.score ratio = submission.upvote_ratio * 100 From b22e3f9828d3443bb2a57feae350a4d9d120c481 Mon Sep 17 00:00:00 2001 From: Jason Date: Thu, 7 Jul 2022 21:25:36 -0400 Subject: [PATCH 090/126] fixes streamlabs_polly ratelimit --- TTS/streamlabs_polly.py | 21 ++++++++------ utils/voice.py | 61 +++++++++++++++++++++++++++++++++++++++++ 2 files changed, 74 insertions(+), 8 deletions(-) diff --git a/TTS/streamlabs_polly.py b/TTS/streamlabs_polly.py index 1f8a039..b7365ab 100644 --- a/TTS/streamlabs_polly.py +++ b/TTS/streamlabs_polly.py @@ -2,6 +2,7 @@ import random import requests from requests.exceptions import JSONDecodeError from utils import settings +from utils.voice import check_ratelimit voices = [ "Brian", @@ -42,16 +43,20 @@ class StreamlabsPolly: voice = str(settings.config["settings"]["tts"]["streamlabs_polly_voice"]).capitalize() body = {"voice": voice, "text": text, "service": "polly"} response = requests.post(self.url, data=body) - try: - voice_data = requests.get(response.json()["speak_url"]) - with open(filepath, "wb") as f: - f.write(voice_data.content) - except (KeyError, JSONDecodeError): + if not check_ratelimit(response): + self.run(text, filepath, random_voice) + + else: try: - if response.json()["error"] == "No text specified!": - raise ValueError("Please specify a text to convert to speech.") + voice_data = requests.get(response.json()["speak_url"]) + with open(filepath, "wb") as f: + f.write(voice_data.content) except (KeyError, JSONDecodeError): - print("Error occurred calling Streamlabs Polly") + try: + if response.json()["error"] == "No text specified!": + raise ValueError("Please specify a text to convert to speech.") + except (KeyError, JSONDecodeError): + print("Error occurred calling Streamlabs Polly") def randomvoice(self): return random.choice(self.voices) diff --git a/utils/voice.py b/utils/voice.py index c4f27bf..e92f0f4 100644 --- a/utils/voice.py +++ b/utils/voice.py @@ -1,4 +1,65 @@ import re +import sys +from datetime import datetime +import time as pytime +from time import sleep + +from requests import Response + +if sys.version_info[0] >= 3: + from datetime import timezone + + +def check_ratelimit(response: Response): + """ + Checks if the response is a ratelimit response. + If it is, it sleeps for the time specified in the response. + """ + if response.status_code == 429: + try: + time = int(response.headers["X-RateLimit-Reset"]) + print(f"Ratelimit hit. Sleeping for {time - int(pytime.time())} seconds.") + sleep_until(time) + return False + except KeyError: # if the header is not present, we don't know how long to wait + return False + + return True + + +def sleep_until(time): + """ + Pause your program until a specific end time. + 'time' is either a valid datetime object or unix timestamp in seconds (i.e. seconds since Unix epoch) + """ + end = time + + # Convert datetime to unix timestamp and adjust for locality + if isinstance(time, datetime): + # If we're on Python 3 and the user specified a timezone, convert to UTC and get tje timestamp. + if sys.version_info[0] >= 3 and time.tzinfo: + end = time.astimezone(timezone.utc).timestamp() + else: + zoneDiff = pytime.time() - (datetime.now() - datetime(1970, 1, 1)).total_seconds() + end = (time - datetime(1970, 1, 1)).total_seconds() + zoneDiff + + # Type check + if not isinstance(end, (int, float)): + raise Exception('The time parameter is not a number or datetime object') + + # Now we wait + while True: + now = pytime.time() + diff = end - now + + # + # Time is up! + # + if diff <= 0: + break + else: + # 'logarithmic' sleeping to minimize loop iterations + sleep(diff / 2) def sanitize_text(text: str) -> str: From d05625d212940c5db4e76ddd18bc51481ff8ae32 Mon Sep 17 00:00:00 2001 From: Jason Date: Thu, 7 Jul 2022 21:27:12 -0400 Subject: [PATCH 091/126] feat: partial background audio implementation #11 currently doesn't fully work (note: not my finest work, I wrote this sleep-deprived) I need to push for other changes --- video_creation/final_video.py | 10 ++++++---- 1 file changed, 6 insertions(+), 4 deletions(-) diff --git a/video_creation/final_video.py b/video_creation/final_video.py index eed072a..d514703 100755 --- a/video_creation/final_video.py +++ b/video_creation/final_video.py @@ -130,9 +130,10 @@ def make_final_video( verbose=False, threads=multiprocessing.cpu_count(), ) - if settings.config["settings"]["background_audio"]: # background.mp3 - if not exists(f"assets/mp3/background.mp3"): - print_substep("optional background audio file didn't so skipping.") + if settings.config["settings"]["background_audio"]: + print('[bold green] Merging background audio with video') + if not exists(f"assets/backgrounds/background.mp3"): + print_substep("Cannot find assets/backgrounds/background.mp3 audio file didn't so skipping.") ffmpeg_extract_subclip( "assets/temp/temp.mp4", 0, @@ -141,13 +142,14 @@ def make_final_video( ) else: ffmpeg_merge_video_audio("assets/temp/temp.mp4", "assets/backgrounds/background.mp3", "assets/temp/temp_audio.mp4") - ffmpeg_extract_subclip( + ffmpeg_extract_subclip( # check if this gets run "assets/temp/temp_audio.mp4", 0, final.duration, targetname=f"results/{subreddit}/{filename}", ) else: + print('debug duck') ffmpeg_extract_subclip( "assets/temp/temp.mp4", 0, From dab3e64fef0eaf4ff06a434967cda0d309fb6718 Mon Sep 17 00:00:00 2001 From: Jason Date: Thu, 7 Jul 2022 21:35:41 -0400 Subject: [PATCH 092/126] style: reformatted. --- utils/subreddit.py | 4 ++-- utils/voice.py | 2 +- video_creation/final_video.py | 16 +++++++++++----- 3 files changed, 14 insertions(+), 8 deletions(-) diff --git a/utils/subreddit.py b/utils/subreddit.py index 9c6ef31..48dceba 100644 --- a/utils/subreddit.py +++ b/utils/subreddit.py @@ -16,8 +16,8 @@ def get_subreddit_undone(submissions: list, subreddit): Any: The submission that has not been done """ # recursively checks if the top submission in the list was already done. - if not exists('./video_creation/data/videos.json'): - with open('./video_creation/data/videos.json', 'w+') as f: + if not exists("./video_creation/data/videos.json"): + with open("./video_creation/data/videos.json", "w+") as f: json.dump([], f) with open("./video_creation/data/videos.json", "r", encoding="utf-8") as done_vids_raw: done_videos = json.load(done_vids_raw) diff --git a/utils/voice.py b/utils/voice.py index e92f0f4..0272b09 100644 --- a/utils/voice.py +++ b/utils/voice.py @@ -45,7 +45,7 @@ def sleep_until(time): # Type check if not isinstance(end, (int, float)): - raise Exception('The time parameter is not a number or datetime object') + raise Exception("The time parameter is not a number or datetime object") # Now we wait while True: diff --git a/video_creation/final_video.py b/video_creation/final_video.py index 920f11b..f1e1f96 100755 --- a/video_creation/final_video.py +++ b/video_creation/final_video.py @@ -131,9 +131,11 @@ def make_final_video( threads=multiprocessing.cpu_count(), ) if settings.config["settings"]["background_audio"]: - print('[bold green] Merging background audio with video') + print("[bold green] Merging background audio with video") if not exists(f"assets/backgrounds/background.mp3"): - print_substep("Cannot find assets/backgrounds/background.mp3 audio file didn't so skipping.") + print_substep( + "Cannot find assets/backgrounds/background.mp3 audio file didn't so skipping." + ) ffmpeg_extract_subclip( "assets/temp/temp.mp4", 0, @@ -141,15 +143,19 @@ def make_final_video( targetname=f"results/{subreddit}/{filename}", ) else: - ffmpeg_merge_video_audio("assets/temp/temp.mp4", "assets/backgrounds/background.mp3", "assets/temp/temp_audio.mp4") - ffmpeg_extract_subclip( # check if this gets run + ffmpeg_merge_video_audio( + "assets/temp/temp.mp4", + "assets/backgrounds/background.mp3", + "assets/temp/temp_audio.mp4", + ) + ffmpeg_extract_subclip( # check if this gets run "assets/temp/temp_audio.mp4", 0, final.duration, targetname=f"results/{subreddit}/{filename}", ) else: - print('debug duck') + print("debug duck") ffmpeg_extract_subclip( "assets/temp/temp.mp4", 0, From 36488a203fff9a0d86008e25fef69de2d558d639 Mon Sep 17 00:00:00 2001 From: Callum Leslie Date: Fri, 8 Jul 2022 02:31:11 +0100 Subject: [PATCH 093/126] chore: delete unused file --- utils/loader.py | 51 ------------------------------------------------- 1 file changed, 51 deletions(-) delete mode 100644 utils/loader.py diff --git a/utils/loader.py b/utils/loader.py deleted file mode 100644 index b9dc276..0000000 --- a/utils/loader.py +++ /dev/null @@ -1,51 +0,0 @@ -# Okay, have to admit. This code is from StackOverflow. It's so efficient, that it's probably the best way to do it. -# Although, it is edited to use less threads. - - -from itertools import cycle -from shutil import get_terminal_size -from threading import Thread -from time import sleep - - -class Loader: - def __init__(self, desc="Loading...", end="Done!", timeout=0.1): - """ - A loader-like context manager - - Args: - desc (str, optional): The loader's description. Defaults to "Loading...". - end (str, optional): Final print. Defaults to "Done!". - timeout (float, optional): Sleep time between prints. Defaults to 0.1. - """ - self.desc = desc - self.end = end - self.timeout = timeout - - self._thread = Thread(target=self._animate, daemon=True) - self.steps = ["⢿", "⣻", "⣽", "⣾", "⣷", "⣯", "⣟", "⡿"] - self.done = False - - def start(self): - self._thread.start() - return self - - def _animate(self): - for c in cycle(self.steps): - if self.done: - break - print(f"\r{self.desc} {c}", flush=True, end="") - sleep(self.timeout) - - def __enter__(self): - self.start() - - def stop(self): - self.done = True - cols = get_terminal_size((80, 20)).columns - print("\r" + " " * cols, end="", flush=True) - print(f"\r{self.end}", flush=True) - - def __exit__(self, exc_type, exc_value, tb): - # handle exceptions with those variables ^ - self.stop() From e1d4c6004ea549c28aab73d09e8880bf02efdb7b Mon Sep 17 00:00:00 2001 From: Jason Date: Thu, 7 Jul 2022 22:46:44 -0400 Subject: [PATCH 094/126] tests a fix stale.yml --- .github/workflows/stale.yml | 12 ++++++++---- 1 file changed, 8 insertions(+), 4 deletions(-) diff --git a/.github/workflows/stale.yml b/.github/workflows/stale.yml index b2f6cac..75bfe4c 100644 --- a/.github/workflows/stale.yml +++ b/.github/workflows/stale.yml @@ -14,14 +14,16 @@ jobs: with: stale-issue-message: 'This issue is stale because it has been open 7 days with no activity. Remove stale label or comment, or this will be closed in 10 days.' close-issue-message: 'Issue closed due to being stale. Please reopen if issue persists in latest version.' - days-before-stale: 7 - days-before-close: 10 + days-before-stale: 6 + days-before-close: 12 stale-issue-label: 'stale' close-issue-label: 'outdated' exempt-issue-labels: 'enhancement,keep,blocked' exempt-all-issue-milestones: true operations-per-run: 300 remove-stale-when-updated: true + ACTIONS_STEP_DEBUG: true + debug-only: true - uses: actions/stale@main id: stale-pr @@ -29,11 +31,13 @@ jobs: with: stale-pr-message: 'This pull request is stale as it has been open for 7 days with no activity. Remove stale label or comment, or this will be closed in 10 days.' close-pr-message: 'Pull request closed due to being stale.' - days-before-stale: 7 - days-before-close: 10 + days-before-stale: 10 + days-before-close: 20 close-pr-label: 'outdated' stale-pr-label: 'stale' exempt-pr-labels: 'keep,blocked,before next release,after next release' exempt-all-pr-milestones: true operations-per-run: 300 remove-stale-when-updated: true + ACTIONS_STEP_DEBUG: true + debug-only: true From 10a89f07579ab54c2d5021210d6620ca21136eb2 Mon Sep 17 00:00:00 2001 From: Jason Date: Thu, 7 Jul 2022 22:58:38 -0400 Subject: [PATCH 095/126] tests a fix stale.yml --- .github/workflows/stale.yml | 14 ++++++++++---- 1 file changed, 10 insertions(+), 4 deletions(-) diff --git a/.github/workflows/stale.yml b/.github/workflows/stale.yml index 75bfe4c..1286d81 100644 --- a/.github/workflows/stale.yml +++ b/.github/workflows/stale.yml @@ -8,10 +8,12 @@ jobs: stale: runs-on: ubuntu-latest steps: - - uses: actions/stale@main + - uses: actions/stale@v4 id: stale-issue name: stale-issue with: + # general settings + repo-token: ${{ secrets.GITHUB_TOKEN }} stale-issue-message: 'This issue is stale because it has been open 7 days with no activity. Remove stale label or comment, or this will be closed in 10 days.' close-issue-message: 'Issue closed due to being stale. Please reopen if issue persists in latest version.' days-before-stale: 6 @@ -21,14 +23,18 @@ jobs: exempt-issue-labels: 'enhancement,keep,blocked' exempt-all-issue-milestones: true operations-per-run: 300 + ascending: true + + # debug remove-stale-when-updated: true - ACTIONS_STEP_DEBUG: true debug-only: true - - uses: actions/stale@main + - uses: actions/stale@v4 id: stale-pr name: stale-pr with: + # general settings + repo-token: ${{ secrets.GITHUB_TOKEN }} stale-pr-message: 'This pull request is stale as it has been open for 7 days with no activity. Remove stale label or comment, or this will be closed in 10 days.' close-pr-message: 'Pull request closed due to being stale.' days-before-stale: 10 @@ -38,6 +44,6 @@ jobs: exempt-pr-labels: 'keep,blocked,before next release,after next release' exempt-all-pr-milestones: true operations-per-run: 300 + # debug remove-stale-when-updated: true - ACTIONS_STEP_DEBUG: true debug-only: true From f0499df4413a80e3b47692b1a33d610630a11ee0 Mon Sep 17 00:00:00 2001 From: Jason Date: Thu, 7 Jul 2022 23:02:49 -0400 Subject: [PATCH 096/126] tests a fix stale.yml --- .github/workflows/stale.yml | 13 +++++++------ 1 file changed, 7 insertions(+), 6 deletions(-) diff --git a/.github/workflows/stale.yml b/.github/workflows/stale.yml index 1286d81..3db14aa 100644 --- a/.github/workflows/stale.yml +++ b/.github/workflows/stale.yml @@ -5,8 +5,12 @@ on: - cron: '0 0 * * *' jobs: + stale: runs-on: ubuntu-latest + permissions: + issues: write + pull-requests: write steps: - uses: actions/stale@v4 id: stale-issue @@ -23,11 +27,9 @@ jobs: exempt-issue-labels: 'enhancement,keep,blocked' exempt-all-issue-milestones: true operations-per-run: 300 - ascending: true - - # debug remove-stale-when-updated: true - debug-only: true + ascending: true + #debug-only: true - uses: actions/stale@v4 id: stale-pr @@ -44,6 +46,5 @@ jobs: exempt-pr-labels: 'keep,blocked,before next release,after next release' exempt-all-pr-milestones: true operations-per-run: 300 - # debug remove-stale-when-updated: true - debug-only: true + #debug-only: true From 6c17f5486aa6054278dbee01fc3eff9017af852d Mon Sep 17 00:00:00 2001 From: micziz <99438936+micziz@users.noreply.github.com> Date: Fri, 8 Jul 2022 21:58:13 +0200 Subject: [PATCH 097/126] Fix repo --- README.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/README.md b/README.md index dad8f59..c0e7095 100644 --- a/README.md +++ b/README.md @@ -46,7 +46,7 @@ The only original thing being done is the editing and gathering of all materials On MacOS and Linux (debian, arch, fedora and centos, and based on those), you can run an install script that will automatically install steps 1 to 3. (requires bash) -`bash <(curl -sL https://raw.githubusercontent.com/micziz/RedditVideoMakerBot/master/install.sh)` +`bash <(curl -sL https://raw.githubusercontent.com/elebumm/RedditVideoMakerBot/master/install.sh)` This can also be used to update the installation From 259c17a38cb36787090f5a998c5e49f16004d644 Mon Sep 17 00:00:00 2001 From: micziz <99438936+micziz@users.noreply.github.com> Date: Fri, 8 Jul 2022 22:01:41 +0200 Subject: [PATCH 098/126] Fix playwright --- install.sh | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/install.sh b/install.sh index 6453dc0..7f24870 100644 --- a/install.sh +++ b/install.sh @@ -114,8 +114,8 @@ function install_playwright(){ # cd into the directory where the script is downloaded cd RedditVideoMakerBot # run the install script - playwright install - playwright install-deps + python3 -m playwright install + python3 -m playwright install-deps # give a note printf "Note, if these gave any errors, please run this command This will (maybe) fix the issue with playwright.\npython -m playwright install && python -m playwright install-deps" cd .. @@ -217,4 +217,4 @@ function install_main(){ } # Run the main function -install_main \ No newline at end of file +install_main From de36ba79e98978f751a8020f27fdd536ac80234e Mon Sep 17 00:00:00 2001 From: Callum Leslie Date: Sat, 9 Jul 2022 09:32:01 +0000 Subject: [PATCH 099/126] fix: add hints for playwright on arch --- install.sh | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/install.sh b/install.sh index 7f24870..4eae0ae 100644 --- a/install.sh +++ b/install.sh @@ -117,7 +117,9 @@ function install_playwright(){ python3 -m playwright install python3 -m playwright install-deps # give a note - printf "Note, if these gave any errors, please run this command This will (maybe) fix the issue with playwright.\npython -m playwright install && python -m playwright install-deps" + printfNote, if these gave any errors, playwright may not be officially supported on your OS, check this issues page for support\nhttps://github.com/microsoft/playwright/issues "Note, if these gave any errors, playwright may not be officially supported on your OS, check this issues page for support\nhttps://github.com/microsoft/playwright/issues\n" + if [ -x "$(command -v pacman)" ]; then + printf "It seems you are on and Arch based distro.\nTry installing these from the AUR for playwright to run:\nenchant1.6\nicu66\nlibwebp052\n" cd .. } From c617af98cef6f4d98d0e8ff3f7cc1f4ffcb00b0f Mon Sep 17 00:00:00 2001 From: Callum Leslie Date: Sat, 9 Jul 2022 12:57:12 +0100 Subject: [PATCH 100/126] fix: closed the if and removed duplicate sentence --- install.sh | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/install.sh b/install.sh index 4eae0ae..782069d 100644 --- a/install.sh +++ b/install.sh @@ -117,9 +117,10 @@ function install_playwright(){ python3 -m playwright install python3 -m playwright install-deps # give a note - printfNote, if these gave any errors, playwright may not be officially supported on your OS, check this issues page for support\nhttps://github.com/microsoft/playwright/issues "Note, if these gave any errors, playwright may not be officially supported on your OS, check this issues page for support\nhttps://github.com/microsoft/playwright/issues\n" + printf "Note, if these gave any errors, playwright may not be officially supported on your OS, check this issues page for support\nhttps://github.com/microsoft/playwright/issues" if [ -x "$(command -v pacman)" ]; then printf "It seems you are on and Arch based distro.\nTry installing these from the AUR for playwright to run:\nenchant1.6\nicu66\nlibwebp052\n" + fi cd .. } From 29de549ef3c1ca4b181d905e8a9a70803a3426ac Mon Sep 17 00:00:00 2001 From: Callum Leslie Date: Mon, 11 Jul 2022 17:52:12 +0100 Subject: [PATCH 101/126] fix: skip comments that are blank after sanitisation --- reddit/subreddit.py | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/reddit/subreddit.py b/reddit/subreddit.py index b64a52a..c316ccf 100644 --- a/reddit/subreddit.py +++ b/reddit/subreddit.py @@ -7,6 +7,7 @@ from praw.models import MoreComments from utils.console import print_step, print_substep from utils.subreddit import get_subreddit_undone from utils.videos import check_done +from utils.voice import sanitize_text def get_subreddit_threads(POST_ID: str): @@ -95,6 +96,9 @@ def get_subreddit_threads(POST_ID: str): if top_level_comment.body in ["[removed]", "[deleted]"]: continue # # see https://github.com/JasonLovesDoggo/RedditVideoMakerBot/issues/78 if not top_level_comment.stickied: + sanitised = sanitize_text(top_level_comment.body) + if not sanitised or sanitised != " ": + continue if len(top_level_comment.body) <= int( settings.config["reddit"]["thread"]["max_comment_length"] ): From c1c72ca5af65fbb9b7529ad6c7df5cb96b233561 Mon Sep 17 00:00:00 2001 From: Callum Leslie Date: Mon, 11 Jul 2022 17:56:24 +0100 Subject: [PATCH 102/126] fix: add more characters to illegal chars --- utils/voice.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/utils/voice.py b/utils/voice.py index 0272b09..4a77833 100644 --- a/utils/voice.py +++ b/utils/voice.py @@ -81,7 +81,7 @@ def sanitize_text(text: str) -> str: result = re.sub(regex_urls, " ", text) # note: not removing apostrophes - regex_expr = r"\s['|’]|['|’]\s|[\^_~@!&;#:\-%“”‘\"%\*/{}\[\]\(\)\\|<>=+]" + regex_expr = r"\s['|’]|['|’]\s|[\^_~@!&;#:\-–—%“”‘\"%\*/{}\[\]\(\)\\|<>=+]" result = re.sub(regex_expr, " ", result) result = result.replace("+", "plus").replace("&", "and") # remove extra whitespace From 302cd355872a8d298f84787f53879c00bef26d47 Mon Sep 17 00:00:00 2001 From: Callum Leslie Date: Mon, 11 Jul 2022 17:58:49 +0100 Subject: [PATCH 103/126] fix: logic error --- reddit/subreddit.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/reddit/subreddit.py b/reddit/subreddit.py index c316ccf..4cfd669 100644 --- a/reddit/subreddit.py +++ b/reddit/subreddit.py @@ -97,7 +97,8 @@ def get_subreddit_threads(POST_ID: str): continue # # see https://github.com/JasonLovesDoggo/RedditVideoMakerBot/issues/78 if not top_level_comment.stickied: sanitised = sanitize_text(top_level_comment.body) - if not sanitised or sanitised != " ": + print(sanitised) + if not sanitised or sanitised == " ": continue if len(top_level_comment.body) <= int( settings.config["reddit"]["thread"]["max_comment_length"] From 721eca4d3e7e745531d68bbf4865624c2592be06 Mon Sep 17 00:00:00 2001 From: Callum Leslie Date: Mon, 11 Jul 2022 18:00:50 +0100 Subject: [PATCH 104/126] chore: remove debug statement --- reddit/subreddit.py | 1 - 1 file changed, 1 deletion(-) diff --git a/reddit/subreddit.py b/reddit/subreddit.py index 4cfd669..829a3a8 100644 --- a/reddit/subreddit.py +++ b/reddit/subreddit.py @@ -97,7 +97,6 @@ def get_subreddit_threads(POST_ID: str): continue # # see https://github.com/JasonLovesDoggo/RedditVideoMakerBot/issues/78 if not top_level_comment.stickied: sanitised = sanitize_text(top_level_comment.body) - print(sanitised) if not sanitised or sanitised == " ": continue if len(top_level_comment.body) <= int( From 7461ddb811b56f44bff3b66e42ed9763783875d0 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Muhammed=20Mustafa=20Ak=C5=9Fam?= Date: Mon, 11 Jul 2022 20:26:34 +0300 Subject: [PATCH 105/126] Create index.html Created index.html file for quick access to video via another device. --- index.html | 281 +++++++++++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 281 insertions(+) create mode 100644 index.html diff --git a/index.html b/index.html new file mode 100644 index 0000000..46ae793 --- /dev/null +++ b/index.html @@ -0,0 +1,281 @@ + + + + + + RedditVideoMakerBot + + + + + + + + + +
+
+
+ + + RedditVideoMakerBot + +
+
+
+ +
+
+
+ +
+
+ +
+
+ +
+ +
+
+
+
+ +
+
+

+ Back to top +

+

Album Example Theme by © Bootstrap.

+

If your data is not refreshing, try to Hard reload(Ctrl + F5) and visit your local videos.json file.

+
+
+ + + + + + + + + + \ No newline at end of file From a3e7b979f3e2833e6c46a651142360691b737024 Mon Sep 17 00:00:00 2001 From: Callum Leslie Date: Mon, 11 Jul 2022 19:36:46 +0100 Subject: [PATCH 106/126] fix: regex splitter no longer skips newlines --- TTS/engine_wrapper.py | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/TTS/engine_wrapper.py b/TTS/engine_wrapper.py index a171db7..df90569 100644 --- a/TTS/engine_wrapper.py +++ b/TTS/engine_wrapper.py @@ -79,7 +79,9 @@ class TTSEngine: split_files = [] split_text = [ x.group().strip() - for x in re.finditer(rf" *((.{{0,{self.tts_module.max_chars}}})(\.|.$))", text) + for x in re.finditer( + r" *(((.|\n){0," + str(self.tts_module.max_chars) + "})(\.|.$))", text + ) ] idy = None From c412155720d3cb6544608e0fed87e364b0a6fa37 Mon Sep 17 00:00:00 2001 From: Jason Date: Fri, 8 Jul 2022 00:50:15 -0400 Subject: [PATCH 107/126] background audio implementation --- .config.template.toml | 5 ++ TTS/engine_wrapper.py | 15 +++-- TTS/streamlabs_polly.py | 4 +- main.py | 2 +- reddit/subreddit.py | 10 ++-- utils/voice.py | 2 +- video_creation/background.py | 2 +- video_creation/final_video.py | 110 +++++++++++----------------------- video_creation/voices.py | 2 +- 9 files changed, 62 insertions(+), 90 deletions(-) diff --git a/.config.template.toml b/.config.template.toml index ddfa293..16a6436 100644 --- a/.config.template.toml +++ b/.config.template.toml @@ -29,10 +29,15 @@ opacity = { optional = false, default = 0.9, example = 0.8, explanation = "Sets storymode = { optional = true, type = "bool", default = false, example = false, options = [true, false, ], explanation = "not yet implemented" } + +[settings.background] background_choice = { optional = true, default = "minecraft", example = "minecraft", options = ["minecraft", "gta", "rocket-league", "motor-gta", ""], explanation = "Sets the background for the video" } background_audio = { optional = true, type = "bool", default = false, example = false, options = [true, false, ], explaination="Sets a audio to play in the background (put a background.mp3 file in the assets/backgrounds directory for it to be used.)" } +background_audio_volume = { optional = true, type = "float", default = 0.3, example = 0.1, explanation="Sets the volume of the background audio. only used if the background_audio is also set to true" } + + [settings.tts] choice = { optional = false, default = "", options = ["streamlabspolly", "tiktok", "googletranslate", "awspolly", ], example = "streamlabspolly", explanation = "The backend used for TTS generation. This can be left blank and you will be prompted to choose at runtime." } aws_polly_voice = { optional = false, default = "Matthew", example = "Matthew", explanation = "The voice used for AWS Polly" } diff --git a/TTS/engine_wrapper.py b/TTS/engine_wrapper.py index df90569..4f3cf1c 100644 --- a/TTS/engine_wrapper.py +++ b/TTS/engine_wrapper.py @@ -47,7 +47,7 @@ class TTSEngine: Path(self.path).mkdir(parents=True, exist_ok=True) - # This file needs to be removed in case this post does not use post text, so that it wont appear in the final video + # This file needs to be removed in case this post does not use post text, so that it won't appear in the final video try: Path(f"{self.path}/posttext.mp3").unlink() except OSError: @@ -67,10 +67,12 @@ class TTSEngine: # ! Stop creating mp3 files if the length is greater than max length. if self.length > self.max_length: break - if not self.tts_module.max_chars: + if ( + len(comment["comment_body"]) > self.tts_module.max_chars + ): # Split the comment if it is too long + self.split_post(comment["comment_body"], idx) # Split the comment + else: # If the comment is not too long, just call the tts engine self.call_tts(f"{idx}", comment["comment_body"]) - else: - self.split_post(comment["comment_body"], idx) print_substep("Saved Text to MP3 files successfully.", style="bold green") return self.length, idx @@ -84,9 +86,12 @@ class TTSEngine: ) ] - idy = None for idy, text_cut in enumerate(split_text): # print(f"{idx}-{idy}: {text_cut}\n") + if text_cut == "": + print("Empty text cut: tell the devs about this") + continue + self.call_tts(f"{idx}-{idy}.part", text_cut) split_files.append(AudioFileClip(f"{self.path}/{idx}-{idy}.part.mp3")) CompositeAudioClip([concatenate_audioclips(split_files)]).write_audiofile( diff --git a/TTS/streamlabs_polly.py b/TTS/streamlabs_polly.py index b7365ab..a5fda6a 100644 --- a/TTS/streamlabs_polly.py +++ b/TTS/streamlabs_polly.py @@ -48,10 +48,12 @@ class StreamlabsPolly: else: try: + print(body) + print(response.json()) voice_data = requests.get(response.json()["speak_url"]) with open(filepath, "wb") as f: f.write(voice_data.content) - except (KeyError, JSONDecodeError): + except (KeyError, JSONDecodeError) as e: try: if response.json()["error"] == "No text specified!": raise ValueError("Please specify a text to convert to speech.") diff --git a/main.py b/main.py index 8ce8725..63e4e84 100755 --- a/main.py +++ b/main.py @@ -32,7 +32,7 @@ print( print_markdown( "### Thanks for using this tool! [Feel free to contribute to this project on GitHub!](https://lewismenelaws.com) If you have any questions, feel free to reach out to me on Twitter or submit a GitHub issue. You can find solutions to many common problems in the [Documentation](https://luka-hietala.gitbook.io/documentation-for-the-reddit-bot/)" ) -print_step(f"You are using V{VERSION} of the bot") +print_step(f"You are using v{VERSION} of the bot") def main(POST_ID=None): diff --git a/reddit/subreddit.py b/reddit/subreddit.py index 829a3a8..716a7fa 100644 --- a/reddit/subreddit.py +++ b/reddit/subreddit.py @@ -18,7 +18,7 @@ def get_subreddit_threads(POST_ID: str): print_substep("Logging into Reddit.") content = {} - if settings.config["reddit"]["creds"]["2fa"] == True: + if settings.config["reddit"]["creds"]["2fa"]: print("\nEnter your two-factor authentication code from your authenticator app.\n") code = input("> ") print() @@ -27,7 +27,7 @@ def get_subreddit_threads(POST_ID: str): else: passkey = settings.config["reddit"]["creds"]["password"] username = settings.config["reddit"]["creds"]["username"] - if username.casefold().startswith("u/"): + if str(username).casefold().startswith("u/"): username = username[2:] reddit = praw.Reddit( client_id=settings.config["reddit"]["creds"]["client_id"], @@ -55,7 +55,7 @@ def get_subreddit_threads(POST_ID: str): sub = settings.config["reddit"]["thread"]["subreddit"] print_substep(f"Using subreddit: r/{sub} from TOML config") subreddit_choice = sub - if subreddit_choice.casefold().startswith("r/"): # removes the r/ from the input + if str(subreddit_choice).casefold().startswith("r/"): # removes the r/ from the input subreddit_choice = subreddit_choice[2:] subreddit = reddit.subreddit( subreddit_choice @@ -65,11 +65,10 @@ def get_subreddit_threads(POST_ID: str): submission = reddit.submission(id=POST_ID) elif ( settings.config["reddit"]["thread"]["post_id"] - and len(settings.config["reddit"]["thread"]["post_id"].split("+")) == 1 + and len(str(settings.config["reddit"]["thread"]["post_id"]).split("+")) == 1 ): submission = reddit.submission(id=settings.config["reddit"]["thread"]["post_id"]) else: - threads = subreddit.hot(limit=25) submission = get_subreddit_undone(threads, subreddit) submission = check_done(submission) # double-checking @@ -104,6 +103,7 @@ def get_subreddit_threads(POST_ID: str): ): if ( top_level_comment.author is not None + and sanitize_text(top_level_comment.body) is not None ): # if errors occur with this change to if not. content["comments"].append( { diff --git a/utils/voice.py b/utils/voice.py index 4a77833..a0709fa 100644 --- a/utils/voice.py +++ b/utils/voice.py @@ -36,7 +36,7 @@ def sleep_until(time): # Convert datetime to unix timestamp and adjust for locality if isinstance(time, datetime): - # If we're on Python 3 and the user specified a timezone, convert to UTC and get tje timestamp. + # If we're on Python 3 and the user specified a timezone, convert to UTC and get the timestamp. if sys.version_info[0] >= 3 and time.tzinfo: end = time.astimezone(timezone.utc).timestamp() else: diff --git a/video_creation/background.py b/video_creation/background.py index 7ef4321..a6efc8b 100644 --- a/video_creation/background.py +++ b/video_creation/background.py @@ -64,7 +64,7 @@ def get_start_and_end_times(video_length: int, length_of_clip: int) -> Tuple[int def get_background_config(): """Fetch the background/s configuration""" try: - choice = str(settings.config["settings"]["background_choice"]).casefold() + choice = str(settings.config["settings"]["background"]["background_choice"]).casefold() except AttributeError: print_substep("No background selected. Picking random background'") choice = None diff --git a/video_creation/final_video.py b/video_creation/final_video.py index f1e1f96..bccbd0a 100755 --- a/video_creation/final_video.py +++ b/video_creation/final_video.py @@ -3,30 +3,22 @@ import multiprocessing import os import re from os.path import exists -from typing import Dict, Tuple, Any - +from typing import Tuple, Any import translators as ts -from moviepy.editor import ( - VideoFileClip, - AudioFileClip, - ImageClip, - concatenate_videoclips, - concatenate_audioclips, - CompositeAudioClip, - CompositeVideoClip, -) +from moviepy.editor import (VideoFileClip, AudioFileClip, ImageClip, concatenate_videoclips, concatenate_audioclips, + CompositeAudioClip, CompositeVideoClip, ) from moviepy.video.io.ffmpeg_tools import ffmpeg_merge_video_audio, ffmpeg_extract_subclip from rich.console import Console +import moviepy.editor as mpe from utils.cleanup import cleanup from utils.console import print_step, print_substep from utils.videos import save_data from utils import settings - console = Console() - +VOLUME_MULTIPLIER = settings.config["settings"]['background']["background_audio_volume"] W, H = 1080, 1920 @@ -48,9 +40,7 @@ def name_normalize(name: str) -> str: return name -def make_final_video( - number_of_clips: int, length: int, reddit_obj: dict, background_config: Tuple[str, str, str, Any] -): +def make_final_video(number_of_clips: int, length: int, reddit_obj: dict, background_config: Tuple[str, str, str, Any]): """Gathers audio clips, gathers all screenshots, stitches them together and saves the final video to assets/temp Args: number_of_clips (int): Index to end at when going through the screenshots' @@ -63,11 +53,8 @@ def make_final_video( VideoFileClip.reH = lambda clip: clip.resize(width=H) opacity = settings.config["settings"]["opacity"] background_clip = ( - VideoFileClip("assets/temp/background.mp4") - .without_audio() - .resize(height=H) - .crop(x1=1166.6, y1=0, x2=2246.6, y2=1920) - ) + VideoFileClip("assets/temp/background.mp4").without_audio().resize(height=H).crop(x1=1166.6, y1=0, x2=2246.6, + y2=1920)) # Gather all audio clips audio_clips = [AudioFileClip(f"assets/temp/mp3/{i}.mp3") for i in range(number_of_clips)] @@ -80,21 +67,13 @@ def make_final_video( image_clips = [] # Gather all images new_opacity = 1 if opacity is None or float(opacity) >= 1 else float(opacity) - image_clips.insert( - 0, - ImageClip("assets/temp/png/title.png") - .set_duration(audio_clips[0].duration) - .resize(width=W - 100) - .set_opacity(new_opacity), - ) + image_clips.insert(0, ImageClip("assets/temp/png/title.png").set_duration(audio_clips[0].duration).resize( + width=W - 100).set_opacity(new_opacity), ) for i in range(0, number_of_clips): image_clips.append( - ImageClip(f"assets/temp/png/comment_{i}.png") - .set_duration(audio_clips[i + 1].duration) - .resize(width=W - 100) - .set_opacity(new_opacity) - ) + ImageClip(f"assets/temp/png/comment_{i}.png").set_duration(audio_clips[i + 1].duration).resize( + width=W - 100).set_opacity(new_opacity)) # if os.path.exists("assets/mp3/posttext.mp3"): # image_clips.insert( @@ -122,51 +101,32 @@ def make_final_video( print_substep("The results folder didn't exist so I made it") os.makedirs(f"./results/{subreddit}") - final.write_videofile( - "assets/temp/temp.mp4", - fps=30, - audio_codec="aac", - audio_bitrate="192k", - verbose=False, - threads=multiprocessing.cpu_count(), - ) - if settings.config["settings"]["background_audio"]: - print("[bold green] Merging background audio with video") - if not exists(f"assets/backgrounds/background.mp3"): - print_substep( - "Cannot find assets/backgrounds/background.mp3 audio file didn't so skipping." - ) - ffmpeg_extract_subclip( - "assets/temp/temp.mp4", - 0, - final.duration, - targetname=f"results/{subreddit}/{filename}", - ) - else: - ffmpeg_merge_video_audio( - "assets/temp/temp.mp4", - "assets/backgrounds/background.mp3", - "assets/temp/temp_audio.mp4", - ) - ffmpeg_extract_subclip( # check if this gets run - "assets/temp/temp_audio.mp4", - 0, - final.duration, - targetname=f"results/{subreddit}/{filename}", - ) + final.write_videofile("assets/temp/temp.mp4", fps=30, audio_codec="aac", audio_bitrate="192k", verbose=False, + threads=multiprocessing.cpu_count(), ) + if settings.config["settings"]['background']["background_audio"] and exists(f"assets/backgrounds/background.mp3"): + if not isinstance(VOLUME_MULTIPLIER, float): + print("No background audio volume set, using default of .3 set it in the config.toml file") + assert VOLUME_MULTIPLIER == float(0.3) + print('Merging background audio with video') + my_clip = mpe.VideoFileClip('assets/temp/temp.mp4') + audio_background = AudioFileClip("assets/backgrounds/background.mp3") + lowered_audio = audio_background.multiply_volume( + VOLUME_MULTIPLIER) # lower volume by background_audio_volume, use with fx + lowered_audio = lowered_audio.subclip(0, my_clip.duration) # trim the audio to the length of the video + lowered_audio.set_duration(my_clip.duration) # set the duration of the audio to the length of the video + final_audio = mpe.CompositeAudioClip([my_clip.audio, lowered_audio]) + final_clip = my_clip.set_audio(final_audio) + + final_clip.write_videofile("assets/temp/temp_audio.mp4", fps=30, audio_codec="aac", audio_bitrate="192k", + verbose=False, threads=multiprocessing.cpu_count()) + ffmpeg_extract_subclip( # check if this gets run + "assets/temp/temp_audio.mp4", 0, final.duration, targetname=f"results/{subreddit}/{filename}", ) else: - print("debug duck") - ffmpeg_extract_subclip( - "assets/temp/temp.mp4", - 0, - final.duration, - targetname=f"results/{subreddit}/{filename}", - ) + ffmpeg_extract_subclip("assets/temp/temp.mp4", 0, final.duration, + targetname=f"results/{subreddit}/{filename}", ) print_step("Removing temporary files 🗑") cleanups = cleanup() print_substep(f"Removed {cleanups} temporary files 🗑") print_substep("See result in the results folder!") - print_step( - f'Reddit title: {reddit_obj["thread_title"]} \n Background Credit: {background_config[2]}' - ) + print_step(f'Reddit title: {reddit_obj["thread_title"]} \n Background Credit: {background_config[2]}') diff --git a/video_creation/voices.py b/video_creation/voices.py index ffc0898..74894eb 100644 --- a/video_creation/voices.py +++ b/video_creation/voices.py @@ -34,7 +34,7 @@ def save_text_to_mp3(reddit_obj) -> Tuple[int, int]: """ voice = settings.config["settings"]["tts"]["choice"] - if voice.casefold() in map(lambda _: _.casefold(), TTSProviders): + if str(voice).casefold() in map(lambda _: _.casefold(), TTSProviders): text_to_mp3 = TTSEngine(get_case_insensitive_key_value(TTSProviders, voice), reddit_obj) else: while True: From aeafb53da3368a2b1975ed93fc7376074b9b45c0 Mon Sep 17 00:00:00 2001 From: Jason Date: Sun, 10 Jul 2022 11:44:40 -0400 Subject: [PATCH 108/126] background audio implementation p2. removed debug calls and testing rn style: removed useless comment imports --- TTS/streamlabs_polly.py | 2 -- main.py | 1 - video_creation/final_video.py | 27 +++++++++++++++++++-------- 3 files changed, 19 insertions(+), 11 deletions(-) diff --git a/TTS/streamlabs_polly.py b/TTS/streamlabs_polly.py index a5fda6a..bcfa175 100644 --- a/TTS/streamlabs_polly.py +++ b/TTS/streamlabs_polly.py @@ -48,8 +48,6 @@ class StreamlabsPolly: else: try: - print(body) - print(response.json()) voice_data = requests.get(response.json()["speak_url"]) with open(filepath, "wb") as f: f.write(voice_data.content) diff --git a/main.py b/main.py index 63e4e84..41ae8e1 100755 --- a/main.py +++ b/main.py @@ -7,7 +7,6 @@ from utils.cleanup import cleanup from utils.console import print_markdown, print_step from utils import settings -# from utils.checker import envUpdate from video_creation.background import ( download_background, chop_background_video, diff --git a/video_creation/final_video.py b/video_creation/final_video.py index bccbd0a..2e774c4 100755 --- a/video_creation/final_video.py +++ b/video_creation/final_video.py @@ -4,11 +4,15 @@ import os import re from os.path import exists from typing import Tuple, Any -import translators as ts - -from moviepy.editor import (VideoFileClip, AudioFileClip, ImageClip, concatenate_videoclips, concatenate_audioclips, - CompositeAudioClip, CompositeVideoClip, ) -from moviepy.video.io.ffmpeg_tools import ffmpeg_merge_video_audio, ffmpeg_extract_subclip +from moviepy import * +from moviepy.audio.AudioClip import concatenate_audioclips, CompositeAudioClip +from moviepy.audio.fx.audio_normalize import audio_normalize +from moviepy.audio.io.AudioFileClip import AudioFileClip +from moviepy.video.VideoClip import ImageClip +from moviepy.video.compositing.CompositeVideoClip import CompositeVideoClip +from moviepy.video.compositing.concatenate import concatenate_videoclips +from moviepy.video.io.VideoFileClip import VideoFileClip +from moviepy.video.io.ffmpeg_tools import ffmpeg_extract_subclip from rich.console import Console import moviepy.editor as mpe @@ -18,7 +22,10 @@ from utils.videos import save_data from utils import settings console = Console() -VOLUME_MULTIPLIER = settings.config["settings"]['background']["background_audio_volume"] +try: + VOLUME_MULTIPLIER = settings.config["settings"]['background']["background_audio_volume"] +except (TypeError, KeyError): + VOLUME_MULTIPLIER = 1 W, H = 1080, 1920 @@ -32,6 +39,7 @@ def name_normalize(name: str) -> str: lang = settings.config["reddit"]["thread"]["post_lang"] if lang: + import translators as ts print_substep("Translating filename...") translated_name = ts.google(name, to_language=lang) return translated_name @@ -110,8 +118,11 @@ def make_final_video(number_of_clips: int, length: int, reddit_obj: dict, backgr print('Merging background audio with video') my_clip = mpe.VideoFileClip('assets/temp/temp.mp4') audio_background = AudioFileClip("assets/backgrounds/background.mp3") - lowered_audio = audio_background.multiply_volume( - VOLUME_MULTIPLIER) # lower volume by background_audio_volume, use with fx + lowered_audio = audio_background.fl(lambda gf, t: VOLUME_MULTIPLIER * gf(t), + keep_duration=True) # to testr + #lowered_audio = audio_normalize(audio_background) + # lowered_audio = audio_background.multiply_volume( # todo get this to work + # VOLUME_MULTIPLIER) # lower volume by background_audio_volume, use with fx lowered_audio = lowered_audio.subclip(0, my_clip.duration) # trim the audio to the length of the video lowered_audio.set_duration(my_clip.duration) # set the duration of the audio to the length of the video final_audio = mpe.CompositeAudioClip([my_clip.audio, lowered_audio]) From 2b0d66f451cd31f377b685ebedca0929a84080d4 Mon Sep 17 00:00:00 2001 From: Jason Date: Sun, 10 Jul 2022 11:54:51 -0400 Subject: [PATCH 109/126] added backup contingency --- TTS/engine_wrapper.py | 9 ++++++--- 1 file changed, 6 insertions(+), 3 deletions(-) diff --git a/TTS/engine_wrapper.py b/TTS/engine_wrapper.py index 4f3cf1c..160c61a 100644 --- a/TTS/engine_wrapper.py +++ b/TTS/engine_wrapper.py @@ -114,9 +114,12 @@ class TTSEngine: # self.length += MP3(f"{self.path}/{filename}.mp3").info.length # except (MutagenError, HeaderNotFoundError): # self.length += sox.file_info.duration(f"{self.path}/{filename}.mp3") - clip = AudioFileClip(f"{self.path}/{filename}.mp3") - self.length += clip.duration - clip.close() + try: + clip = AudioFileClip(f"{self.path}/{filename}.mp3") + self.length += clip.duration + clip.close() + except: + self.length = 0 def process_text(text: str): From b027679c011b1b4c38d87e6acc5a953612408d41 Mon Sep 17 00:00:00 2001 From: Jason Date: Mon, 11 Jul 2022 19:58:29 -0400 Subject: [PATCH 110/126] I give up on #11 can't implement --- .config.template.toml | 8 +++--- video_creation/background.py | 2 +- video_creation/final_video.py | 50 ++++++++++++----------------------- 3 files changed, 22 insertions(+), 38 deletions(-) diff --git a/.config.template.toml b/.config.template.toml index 16a6436..1060a09 100644 --- a/.config.template.toml +++ b/.config.template.toml @@ -32,10 +32,10 @@ storymode = { optional = true, type = "bool", default = false, example = false, [settings.background] background_choice = { optional = true, default = "minecraft", example = "minecraft", options = ["minecraft", "gta", "rocket-league", "motor-gta", ""], explanation = "Sets the background for the video" } -background_audio = { optional = true, type = "bool", default = false, example = false, options = [true, - false, -], explaination="Sets a audio to play in the background (put a background.mp3 file in the assets/backgrounds directory for it to be used.)" } -background_audio_volume = { optional = true, type = "float", default = 0.3, example = 0.1, explanation="Sets the volume of the background audio. only used if the background_audio is also set to true" } +#background_audio = { optional = true, type = "bool", default = false, example = false, options = [true, +# false, +#], explaination="Sets a audio to play in the background (put a background.mp3 file in the assets/backgrounds directory for it to be used.)" } +#background_audio_volume = { optional = true, type = "float", default = 0.3, example = 0.1, explanation="Sets the volume of the background audio. only used if the background_audio is also set to true" } [settings.tts] diff --git a/video_creation/background.py b/video_creation/background.py index a6efc8b..be0f46c 100644 --- a/video_creation/background.py +++ b/video_creation/background.py @@ -30,7 +30,7 @@ background_options = { "https://www.youtube.com/watch?v=2X9QGY__0II", "rocket_league.mp4", "Orbital Gameplay", - "top", + lambda t: ("center", 200 + t), ), "minecraft": ( # Minecraft parkour "https://www.youtube.com/watch?v=n_Dv4JMiwK8", diff --git a/video_creation/final_video.py b/video_creation/final_video.py index 2e774c4..de53a7a 100755 --- a/video_creation/final_video.py +++ b/video_creation/final_video.py @@ -4,9 +4,7 @@ import os import re from os.path import exists from typing import Tuple, Any -from moviepy import * from moviepy.audio.AudioClip import concatenate_audioclips, CompositeAudioClip -from moviepy.audio.fx.audio_normalize import audio_normalize from moviepy.audio.io.AudioFileClip import AudioFileClip from moviepy.video.VideoClip import ImageClip from moviepy.video.compositing.CompositeVideoClip import CompositeVideoClip @@ -14,7 +12,6 @@ from moviepy.video.compositing.concatenate import concatenate_videoclips from moviepy.video.io.VideoFileClip import VideoFileClip from moviepy.video.io.ffmpeg_tools import ffmpeg_extract_subclip from rich.console import Console -import moviepy.editor as mpe from utils.cleanup import cleanup from utils.console import print_step, print_substep @@ -22,10 +19,6 @@ from utils.videos import save_data from utils import settings console = Console() -try: - VOLUME_MULTIPLIER = settings.config["settings"]['background']["background_audio_volume"] -except (TypeError, KeyError): - VOLUME_MULTIPLIER = 1 W, H = 1080, 1920 @@ -56,6 +49,12 @@ def make_final_video(number_of_clips: int, length: int, reddit_obj: dict, backgr reddit_obj (dict): The reddit object that contains the posts to read. background_config (Tuple[str, str, str, Any]): The background config to use. """ + #try: # if it isn't found (i.e you just updated and copied over config.toml) it will throw an error + # VOLUME_MULTIPLIER = settings.config["settings"]['background']["background_audio_volume"] + #except (TypeError, KeyError): + # print('No background audio volume found in config.toml. Using default value of 1.') + # VOLUME_MULTIPLIER = 1 + print_step("Creating the final video 🎥") VideoFileClip.reW = lambda clip: clip.resize(width=W) VideoFileClip.reH = lambda clip: clip.resize(width=H) @@ -103,38 +102,23 @@ def make_final_video(number_of_clips: int, length: int, reddit_obj: dict, backgr filename = f"{name_normalize(title)}.mp4" subreddit = settings.config["reddit"]["thread"]["subreddit"] - save_data(subreddit, filename, title, idx, background_config[2]) - if not exists(f"./results/{subreddit}"): print_substep("The results folder didn't exist so I made it") os.makedirs(f"./results/{subreddit}") + #if settings.config["settings"]['background']["background_audio"] and exists(f"assets/backgrounds/background.mp3"): + # audioclip = mpe.AudioFileClip(f"assets/backgrounds/background.mp3").set_duration(final.duration) + # audioclip = audioclip.fx( volumex, 0.2) + # final_audio = mpe.CompositeAudioClip([final.audio, audioclip]) + # # lowered_audio = audio_background.multiply_volume( # todo get this to work + # # VOLUME_MULTIPLIER) # lower volume by background_audio_volume, use with fx + # final.set_audio(final_audio) + final.write_videofile("assets/temp/temp.mp4", fps=30, audio_codec="aac", audio_bitrate="192k", verbose=False, threads=multiprocessing.cpu_count(), ) - if settings.config["settings"]['background']["background_audio"] and exists(f"assets/backgrounds/background.mp3"): - if not isinstance(VOLUME_MULTIPLIER, float): - print("No background audio volume set, using default of .3 set it in the config.toml file") - assert VOLUME_MULTIPLIER == float(0.3) - print('Merging background audio with video') - my_clip = mpe.VideoFileClip('assets/temp/temp.mp4') - audio_background = AudioFileClip("assets/backgrounds/background.mp3") - lowered_audio = audio_background.fl(lambda gf, t: VOLUME_MULTIPLIER * gf(t), - keep_duration=True) # to testr - #lowered_audio = audio_normalize(audio_background) - # lowered_audio = audio_background.multiply_volume( # todo get this to work - # VOLUME_MULTIPLIER) # lower volume by background_audio_volume, use with fx - lowered_audio = lowered_audio.subclip(0, my_clip.duration) # trim the audio to the length of the video - lowered_audio.set_duration(my_clip.duration) # set the duration of the audio to the length of the video - final_audio = mpe.CompositeAudioClip([my_clip.audio, lowered_audio]) - final_clip = my_clip.set_audio(final_audio) - - final_clip.write_videofile("assets/temp/temp_audio.mp4", fps=30, audio_codec="aac", audio_bitrate="192k", - verbose=False, threads=multiprocessing.cpu_count()) - ffmpeg_extract_subclip( # check if this gets run - "assets/temp/temp_audio.mp4", 0, final.duration, targetname=f"results/{subreddit}/{filename}", ) - else: - ffmpeg_extract_subclip("assets/temp/temp.mp4", 0, final.duration, - targetname=f"results/{subreddit}/{filename}", ) + ffmpeg_extract_subclip("assets/temp/temp.mp4", 0, final.duration, + targetname=f"results/{subreddit}/{filename}", ) + save_data(subreddit, filename, title, idx, background_config[2]) print_step("Removing temporary files 🗑") cleanups = cleanup() print_substep(f"Removed {cleanups} temporary files 🗑") From b2c9f30347b4ab2fc34c0eccd848e552347dd834 Mon Sep 17 00:00:00 2001 From: Jason Date: Mon, 11 Jul 2022 20:27:18 -0400 Subject: [PATCH 111/126] moved index.html and added devs section --- index.html => GUI/index.html | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) rename index.html => GUI/index.html (96%) diff --git a/index.html b/GUI/index.html similarity index 96% rename from index.html rename to GUI/index.html index 46ae793..807c9e7 100644 --- a/index.html +++ b/GUI/index.html @@ -59,7 +59,7 @@ white-space: nowrap; -webkit-overflow-scrolling: touch; } - + #tooltip { background-color: #333; color: white; @@ -105,8 +105,8 @@

Back to top

-

Album Example Theme by © Bootstrap.

-

If your data is not refreshing, try to Hard reload(Ctrl + F5) and visit your local videos.json file.

+

Album Example Theme by © Bootstrap. Developers and Maintainers

+

If your data is not refreshing, try to hard reload(Ctrl + F5) and visit your local videos.json file.

@@ -134,7 +134,7 @@ } $(document).ready(function () { - $.getJSON("video_creation/data/videos.json", + $.getJSON("../video_creation/data/videos.json", function (data) { data.sort((b, a) => a['time'] - b['time']) var video = ''; @@ -163,7 +163,7 @@ video += ''; }); - + $('#videos').append(video); }); }); @@ -278,4 +278,4 @@ } - \ No newline at end of file + From 482a666941ed91f9f33ab7510e271241d1322805 Mon Sep 17 00:00:00 2001 From: Jason Date: Mon, 11 Jul 2022 20:28:32 -0400 Subject: [PATCH 112/126] adds a min comment's var fixes #924 chore: updated version number --- .config.template.toml | 2 +- main.py | 2 +- utils/subreddit.py | 3 +++ video_creation/final_video.py | 1 - video_creation/voices.py | 1 - 5 files changed, 5 insertions(+), 4 deletions(-) diff --git a/.config.template.toml b/.config.template.toml index 1060a09..32c0f7a 100644 --- a/.config.template.toml +++ b/.config.template.toml @@ -16,7 +16,7 @@ subreddit = { optional = false, regex = "[_0-9a-zA-Z]+$", nmin = 3, nmax = 21, e post_id = { optional = true, default = "", regex = "^((?!://|://)[+a-zA-Z])*$", explanation = "Used if you want to use a specific post.", example = "urdtfx" } max_comment_length = { default = 500, optional = false, nmin = 10, nmax = 10000, type = "int", explanation = "max number of characters a comment can have. default is 500", example = 500, oob_error = "the max comment length should be between 10 and 10000" } post_lang = { default = "", optional = true, explanation = "The language you would like to translate to.", example = "es-cr" } - +min_comments = { default = 20, optional = false, nmin = 15, nmax = 999999, type = "int", explanation = "The minimum number of comments a post should have to be included. default is 20", example = 29, oob_error = "the minimum number of comments should be between 15 and 999999" } [settings] allow_nsfw = { optional = false, type = "bool", default = false, example = false, options = [true, false, diff --git a/main.py b/main.py index 41ae8e1..0dbd2c9 100755 --- a/main.py +++ b/main.py @@ -16,7 +16,7 @@ from video_creation.final_video import make_final_video from video_creation.screenshot_downloader import download_screenshots_of_reddit_posts from video_creation.voices import save_text_to_mp3 -VERSION = "2.2.9" +VERSION = "2.2.10" print( """ ██████╗ ███████╗██████╗ ██████╗ ██╗████████╗ ██╗ ██╗██╗██████╗ ███████╗ ██████╗ ███╗ ███╗ █████╗ ██╗ ██╗███████╗██████╗ diff --git a/utils/subreddit.py b/utils/subreddit.py index 48dceba..6f9ed49 100644 --- a/utils/subreddit.py +++ b/utils/subreddit.py @@ -34,6 +34,9 @@ def get_subreddit_undone(submissions: list, subreddit): if submission.stickied: print_substep("This post was pinned by moderators. Skipping...") continue + if submission.num_comments < int(settings.config["reddit"]['thread']["min_comments"]): + print_substep(f'This post has under the specified minimum of comments ({settings.config["reddit"]["thread"]["min_comments"]}). Skipping...') + continue return submission print("all submissions have been done going by top submission order") return get_subreddit_undone( diff --git a/video_creation/final_video.py b/video_creation/final_video.py index de53a7a..264c5af 100755 --- a/video_creation/final_video.py +++ b/video_creation/final_video.py @@ -54,7 +54,6 @@ def make_final_video(number_of_clips: int, length: int, reddit_obj: dict, backgr #except (TypeError, KeyError): # print('No background audio volume found in config.toml. Using default value of 1.') # VOLUME_MULTIPLIER = 1 - print_step("Creating the final video 🎥") VideoFileClip.reW = lambda clip: clip.resize(width=W) VideoFileClip.reH = lambda clip: clip.resize(width=H) diff --git a/video_creation/voices.py b/video_creation/voices.py index 74894eb..ac33dd7 100644 --- a/video_creation/voices.py +++ b/video_creation/voices.py @@ -45,7 +45,6 @@ def save_text_to_mp3(reddit_obj) -> Tuple[int, int]: break print("Unknown Choice") text_to_mp3 = TTSEngine(get_case_insensitive_key_value(TTSProviders, choice), reddit_obj) - return text_to_mp3.run() From 36e07c4c37bc7b368640585a60d3866b0b2d8f77 Mon Sep 17 00:00:00 2001 From: Jason Date: Mon, 11 Jul 2022 20:30:37 -0400 Subject: [PATCH 113/126] see https://github.com/elebumm/RedditVideoMakerBot/issues/924#issuecomment-1181169525 fixes #11 --- utils/subreddit.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/utils/subreddit.py b/utils/subreddit.py index 6f9ed49..cdaf4a6 100644 --- a/utils/subreddit.py +++ b/utils/subreddit.py @@ -34,7 +34,7 @@ def get_subreddit_undone(submissions: list, subreddit): if submission.stickied: print_substep("This post was pinned by moderators. Skipping...") continue - if submission.num_comments < int(settings.config["reddit"]['thread']["min_comments"]): + if submission.num_comments <= int(settings.config["reddit"]['thread']["min_comments"]): print_substep(f'This post has under the specified minimum of comments ({settings.config["reddit"]["thread"]["min_comments"]}). Skipping...') continue return submission From 882bf64e1ca36e7f7a9f964883c309409678fbec Mon Sep 17 00:00:00 2001 From: Jason Date: Mon, 11 Jul 2022 20:30:53 -0400 Subject: [PATCH 114/126] typo --- utils/subreddit.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/utils/subreddit.py b/utils/subreddit.py index cdaf4a6..f80f695 100644 --- a/utils/subreddit.py +++ b/utils/subreddit.py @@ -41,7 +41,7 @@ def get_subreddit_undone(submissions: list, subreddit): print("all submissions have been done going by top submission order") return get_subreddit_undone( subreddit.top(time_filter="hour"), subreddit - ) # all of the videos in hot have already been done + ) # all the videos in hot have already been done def already_done(done_videos: list, submission) -> bool: From 1872ef36c194fc74bcdc6af18179985b80d9b2b5 Mon Sep 17 00:00:00 2001 From: Jason Date: Mon, 11 Jul 2022 20:31:20 -0400 Subject: [PATCH 115/126] reformatted --- utils/subreddit.py | 6 ++-- video_creation/final_video.py | 56 +++++++++++++++++++++++++---------- 2 files changed, 45 insertions(+), 17 deletions(-) diff --git a/utils/subreddit.py b/utils/subreddit.py index f80f695..4eb0108 100644 --- a/utils/subreddit.py +++ b/utils/subreddit.py @@ -34,8 +34,10 @@ def get_subreddit_undone(submissions: list, subreddit): if submission.stickied: print_substep("This post was pinned by moderators. Skipping...") continue - if submission.num_comments <= int(settings.config["reddit"]['thread']["min_comments"]): - print_substep(f'This post has under the specified minimum of comments ({settings.config["reddit"]["thread"]["min_comments"]}). Skipping...') + if submission.num_comments <= int(settings.config["reddit"]["thread"]["min_comments"]): + print_substep( + f'This post has under the specified minimum of comments ({settings.config["reddit"]["thread"]["min_comments"]}). Skipping...' + ) continue return submission print("all submissions have been done going by top submission order") diff --git a/video_creation/final_video.py b/video_creation/final_video.py index 264c5af..28956e6 100755 --- a/video_creation/final_video.py +++ b/video_creation/final_video.py @@ -33,6 +33,7 @@ def name_normalize(name: str) -> str: lang = settings.config["reddit"]["thread"]["post_lang"] if lang: import translators as ts + print_substep("Translating filename...") translated_name = ts.google(name, to_language=lang) return translated_name @@ -41,7 +42,9 @@ def name_normalize(name: str) -> str: return name -def make_final_video(number_of_clips: int, length: int, reddit_obj: dict, background_config: Tuple[str, str, str, Any]): +def make_final_video( + number_of_clips: int, length: int, reddit_obj: dict, background_config: Tuple[str, str, str, Any] +): """Gathers audio clips, gathers all screenshots, stitches them together and saves the final video to assets/temp Args: number_of_clips (int): Index to end at when going through the screenshots' @@ -49,9 +52,9 @@ def make_final_video(number_of_clips: int, length: int, reddit_obj: dict, backgr reddit_obj (dict): The reddit object that contains the posts to read. background_config (Tuple[str, str, str, Any]): The background config to use. """ - #try: # if it isn't found (i.e you just updated and copied over config.toml) it will throw an error + # try: # if it isn't found (i.e you just updated and copied over config.toml) it will throw an error # VOLUME_MULTIPLIER = settings.config["settings"]['background']["background_audio_volume"] - #except (TypeError, KeyError): + # except (TypeError, KeyError): # print('No background audio volume found in config.toml. Using default value of 1.') # VOLUME_MULTIPLIER = 1 print_step("Creating the final video 🎥") @@ -59,8 +62,11 @@ def make_final_video(number_of_clips: int, length: int, reddit_obj: dict, backgr VideoFileClip.reH = lambda clip: clip.resize(width=H) opacity = settings.config["settings"]["opacity"] background_clip = ( - VideoFileClip("assets/temp/background.mp4").without_audio().resize(height=H).crop(x1=1166.6, y1=0, x2=2246.6, - y2=1920)) + VideoFileClip("assets/temp/background.mp4") + .without_audio() + .resize(height=H) + .crop(x1=1166.6, y1=0, x2=2246.6, y2=1920) + ) # Gather all audio clips audio_clips = [AudioFileClip(f"assets/temp/mp3/{i}.mp3") for i in range(number_of_clips)] @@ -73,13 +79,21 @@ def make_final_video(number_of_clips: int, length: int, reddit_obj: dict, backgr image_clips = [] # Gather all images new_opacity = 1 if opacity is None or float(opacity) >= 1 else float(opacity) - image_clips.insert(0, ImageClip("assets/temp/png/title.png").set_duration(audio_clips[0].duration).resize( - width=W - 100).set_opacity(new_opacity), ) + image_clips.insert( + 0, + ImageClip("assets/temp/png/title.png") + .set_duration(audio_clips[0].duration) + .resize(width=W - 100) + .set_opacity(new_opacity), + ) for i in range(0, number_of_clips): image_clips.append( - ImageClip(f"assets/temp/png/comment_{i}.png").set_duration(audio_clips[i + 1].duration).resize( - width=W - 100).set_opacity(new_opacity)) + ImageClip(f"assets/temp/png/comment_{i}.png") + .set_duration(audio_clips[i + 1].duration) + .resize(width=W - 100) + .set_opacity(new_opacity) + ) # if os.path.exists("assets/mp3/posttext.mp3"): # image_clips.insert( @@ -105,7 +119,7 @@ def make_final_video(number_of_clips: int, length: int, reddit_obj: dict, backgr print_substep("The results folder didn't exist so I made it") os.makedirs(f"./results/{subreddit}") - #if settings.config["settings"]['background']["background_audio"] and exists(f"assets/backgrounds/background.mp3"): + # if settings.config["settings"]['background']["background_audio"] and exists(f"assets/backgrounds/background.mp3"): # audioclip = mpe.AudioFileClip(f"assets/backgrounds/background.mp3").set_duration(final.duration) # audioclip = audioclip.fx( volumex, 0.2) # final_audio = mpe.CompositeAudioClip([final.audio, audioclip]) @@ -113,14 +127,26 @@ def make_final_video(number_of_clips: int, length: int, reddit_obj: dict, backgr # # VOLUME_MULTIPLIER) # lower volume by background_audio_volume, use with fx # final.set_audio(final_audio) - final.write_videofile("assets/temp/temp.mp4", fps=30, audio_codec="aac", audio_bitrate="192k", verbose=False, - threads=multiprocessing.cpu_count(), ) - ffmpeg_extract_subclip("assets/temp/temp.mp4", 0, final.duration, - targetname=f"results/{subreddit}/{filename}", ) + final.write_videofile( + "assets/temp/temp.mp4", + fps=30, + audio_codec="aac", + audio_bitrate="192k", + verbose=False, + threads=multiprocessing.cpu_count(), + ) + ffmpeg_extract_subclip( + "assets/temp/temp.mp4", + 0, + final.duration, + targetname=f"results/{subreddit}/{filename}", + ) save_data(subreddit, filename, title, idx, background_config[2]) print_step("Removing temporary files 🗑") cleanups = cleanup() print_substep(f"Removed {cleanups} temporary files 🗑") print_substep("See result in the results folder!") - print_step(f'Reddit title: {reddit_obj["thread_title"]} \n Background Credit: {background_config[2]}') + print_step( + f'Reddit title: {reddit_obj["thread_title"]} \n Background Credit: {background_config[2]}' + ) From a7c54f48e864bd05f9b69ee5f8058b1de19ed17d Mon Sep 17 00:00:00 2001 From: Jason Date: Mon, 11 Jul 2022 20:42:37 -0400 Subject: [PATCH 116/126] Improved AWS Polly eh improved handling for errors like like #929 --- TTS/aws_polly.py | 64 +++++++++++++++++++++++++++--------------------- 1 file changed, 36 insertions(+), 28 deletions(-) diff --git a/TTS/aws_polly.py b/TTS/aws_polly.py index 13eaea1..e1d5318 100644 --- a/TTS/aws_polly.py +++ b/TTS/aws_polly.py @@ -1,6 +1,6 @@ #!/usr/bin/env python3 from boto3 import Session -from botocore.exceptions import BotoCoreError, ClientError +from botocore.exceptions import BotoCoreError, ClientError, ProfileNotFound import sys from utils import settings import random @@ -30,36 +30,44 @@ class AWSPolly: self.voices = voices def run(self, text, filepath, random_voice: bool = False): - session = Session(profile_name="polly") - polly = session.client("polly") - if random_voice: - voice = self.randomvoice() - else: - if not settings.config["settings"]["tts"]["aws_polly_voice"]: - return ValueError( - f"Please set the TOML variable AWS_VOICE to a valid voice. options are: {voices}" - ) - voice = str(settings.config["settings"]["tts"]["aws_polly_voice"]).capitalize() try: - # Request speech synthesis - response = polly.synthesize_speech( - Text=text, OutputFormat="mp3", VoiceId=voice, Engine="neural" - ) - except (BotoCoreError, ClientError) as error: - # The service returned an error, exit gracefully - print(error) - sys.exit(-1) + session = Session(profile_name="polly") + polly = session.client("polly") + if random_voice: + voice = self.randomvoice() + else: + if not settings.config["settings"]["tts"]["aws_polly_voice"]: + return ValueError( + f"Please set the TOML variable AWS_VOICE to a valid voice. options are: {voices}" + ) + voice = str(settings.config["settings"]["tts"]["aws_polly_voice"]).capitalize() + try: + # Request speech synthesis + response = polly.synthesize_speech( + Text=text, OutputFormat="mp3", VoiceId=voice, Engine="neural" + ) + except (BotoCoreError, ClientError) as error: + # The service returned an error, exit gracefully + print(error) + sys.exit(-1) - # Access the audio stream from the response - if "AudioStream" in response: - file = open(filepath, "wb") - file.write(response["AudioStream"].read()) - file.close() - # print_substep(f"Saved Text {idx} to MP3 files successfully.", style="bold green") + # Access the audio stream from the response + if "AudioStream" in response: + file = open(filepath, "wb") + file.write(response["AudioStream"].read()) + file.close() + # print_substep(f"Saved Text {idx} to MP3 files successfully.", style="bold green") - else: - # The response didn't contain audio data, exit gracefully - print("Could not stream audio") + else: + # The response didn't contain audio data, exit gracefully + print("Could not stream audio") + sys.exit(-1) + except ProfileNotFound: + print("You need to install the AWS CLI and configure your profile") + print(""" + Linux: https://docs.aws.amazon.com/polly/latest/dg/setup-aws-cli.html + Windows: https://docs.aws.amazon.com/polly/latest/dg/install-voice-plugin2.html + """) sys.exit(-1) def randomvoice(self): From 49cecb0811ff7b8a7b962ddf6f9ae3e450a02a3d Mon Sep 17 00:00:00 2001 From: Jason Date: Mon, 11 Jul 2022 20:56:24 -0400 Subject: [PATCH 117/126] removed hard cap on subreddit length --- .config.template.toml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.config.template.toml b/.config.template.toml index 32c0f7a..18f1b00 100644 --- a/.config.template.toml +++ b/.config.template.toml @@ -12,11 +12,11 @@ password = { optional = false, nmin = 8, explanation = "the password of your red random = { optional = true, options = [true, false, ], default = false, type = "bool", explanation = "If set to no, it will ask you a thread link to extract the thread, if yes it will randomize it. Default: 'False'", example = "True" } -subreddit = { optional = false, regex = "[_0-9a-zA-Z]+$", nmin = 3, nmax = 21, explanation = "what subreddit to pull posts from, the name of the sub, not the URL", example = "AskReddit", oob_error = "A subreddit name HAS to be between 3 and 20 characters" } +subreddit = { optional = false, regex = "[_0-9a-zA-Z]+$", nmin = 3, explanation = "what subreddit to pull posts from, the name of the sub, not the URL", example = "AskReddit", oob_error = "A subreddit name HAS to be between 3 and 20 characters" } post_id = { optional = true, default = "", regex = "^((?!://|://)[+a-zA-Z])*$", explanation = "Used if you want to use a specific post.", example = "urdtfx" } max_comment_length = { default = 500, optional = false, nmin = 10, nmax = 10000, type = "int", explanation = "max number of characters a comment can have. default is 500", example = 500, oob_error = "the max comment length should be between 10 and 10000" } post_lang = { default = "", optional = true, explanation = "The language you would like to translate to.", example = "es-cr" } -min_comments = { default = 20, optional = false, nmin = 15, nmax = 999999, type = "int", explanation = "The minimum number of comments a post should have to be included. default is 20", example = 29, oob_error = "the minimum number of comments should be between 15 and 999999" } +min_comments = { default = 20, optional = false, nmin = 15, type = "int", explanation = "The minimum number of comments a post should have to be included. default is 20", example = 29, oob_error = "the minimum number of comments should be between 15 and 999999" } [settings] allow_nsfw = { optional = false, type = "bool", default = false, example = false, options = [true, false, From 2f5d394bba68a234610d4cba8512024ec787004d Mon Sep 17 00:00:00 2001 From: Jason Date: Mon, 11 Jul 2022 21:01:29 -0400 Subject: [PATCH 118/126] chore: updated version number --- main.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/main.py b/main.py index 0dbd2c9..b4737e0 100755 --- a/main.py +++ b/main.py @@ -16,7 +16,7 @@ from video_creation.final_video import make_final_video from video_creation.screenshot_downloader import download_screenshots_of_reddit_posts from video_creation.voices import save_text_to_mp3 -VERSION = "2.2.10" +VERSION = "2.3" print( """ ██████╗ ███████╗██████╗ ██████╗ ██╗████████╗ ██╗ ██╗██╗██████╗ ███████╗ ██████╗ ███╗ ███╗ █████╗ ██╗ ██╗███████╗██████╗ From 6528fdc668711f32f6c207527f9972512f378d6f Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Tue, 12 Jul 2022 11:10:30 +0000 Subject: [PATCH 119/126] Bump rich from 12.4.4 to 12.5.1 Bumps [rich](https://github.com/willmcgugan/rich) from 12.4.4 to 12.5.1. - [Release notes](https://github.com/willmcgugan/rich/releases) - [Changelog](https://github.com/Textualize/rich/blob/master/CHANGELOG.md) - [Commits](https://github.com/willmcgugan/rich/compare/v12.4.4...v12.5.1) --- updated-dependencies: - dependency-name: rich dependency-type: direct:production update-type: version-update:semver-minor ... Signed-off-by: dependabot[bot] --- requirements.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/requirements.txt b/requirements.txt index 40c7632..2b7d9eb 100644 --- a/requirements.txt +++ b/requirements.txt @@ -8,5 +8,5 @@ praw==7.6.0 python-dotenv==0.20.0 pytube==12.1.0 requests==2.28.1 -rich==12.4.4 +rich==12.5.1 translators==5.3.1 From c5c64a5e4b6240b61fdbde21609c25ec8694eeed Mon Sep 17 00:00:00 2001 From: Callum Leslie Date: Tue, 12 Jul 2022 16:23:56 +0100 Subject: [PATCH 120/126] fix: dont use blank text splits --- TTS/engine_wrapper.py | 11 ++++++----- 1 file changed, 6 insertions(+), 5 deletions(-) diff --git a/TTS/engine_wrapper.py b/TTS/engine_wrapper.py index 160c61a..c5ed714 100644 --- a/TTS/engine_wrapper.py +++ b/TTS/engine_wrapper.py @@ -85,15 +85,16 @@ class TTSEngine: r" *(((.|\n){0," + str(self.tts_module.max_chars) + "})(\.|.$))", text ) ] - + offset = 0 for idy, text_cut in enumerate(split_text): # print(f"{idx}-{idy}: {text_cut}\n") - if text_cut == "": - print("Empty text cut: tell the devs about this") + if not text_cut or text_cut.isspace(): + offset += 1 continue - self.call_tts(f"{idx}-{idy}.part", text_cut) - split_files.append(AudioFileClip(f"{self.path}/{idx}-{idy}.part.mp3")) + self.call_tts(f"{idx}-{idy - offset}.part", text_cut) + split_files.append(AudioFileClip(f"{self.path}/{idx}-{idy - offset}.part.mp3")) + CompositeAudioClip([concatenate_audioclips(split_files)]).write_audiofile( f"{self.path}/{idx}.mp3", fps=44100, verbose=False, logger=None ) From 89ab71270ab45cdacafc32e95f64de154c8cf8a8 Mon Sep 17 00:00:00 2001 From: Callum Leslie Date: Tue, 12 Jul 2022 16:25:29 +0100 Subject: [PATCH 121/126] style: format --- video_creation/final_video.py | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/video_creation/final_video.py b/video_creation/final_video.py index 28956e6..8524051 100755 --- a/video_creation/final_video.py +++ b/video_creation/final_video.py @@ -43,7 +43,10 @@ def name_normalize(name: str) -> str: def make_final_video( - number_of_clips: int, length: int, reddit_obj: dict, background_config: Tuple[str, str, str, Any] + number_of_clips: int, + length: int, + reddit_obj: dict, + background_config: Tuple[str, str, str, Any], ): """Gathers audio clips, gathers all screenshots, stitches them together and saves the final video to assets/temp Args: From 1d3410646d674334b2169e0e59b1a731c00f1faf Mon Sep 17 00:00:00 2001 From: Jason Date: Tue, 12 Jul 2022 11:40:15 -0400 Subject: [PATCH 122/126] readme: re-added Freebie --- README.md | 2 ++ 1 file changed, 2 insertions(+) diff --git a/README.md b/README.md index 028dcd4..8a9e205 100644 --- a/README.md +++ b/README.md @@ -90,3 +90,5 @@ CallumIO (c.#6837) - https://github.com/CallumIO Verq (Verq#2338) - https://github.com/CordlessCoder LukaHietala (Pix.#0001) - https://github.com/LukaHietala + +Freebiell (Freebie#6429) - https://github.com/FreebieII From 3068cead0c0e4479b93ce3041cc698c92cba96d3 Mon Sep 17 00:00:00 2001 From: Callum Leslie Date: Tue, 12 Jul 2022 16:46:22 +0100 Subject: [PATCH 123/126] fix: raise ValueErrors rather than return --- TTS/aws_polly.py | 8 +++++--- TTS/streamlabs_polly.py | 4 ++-- 2 files changed, 7 insertions(+), 5 deletions(-) diff --git a/TTS/aws_polly.py b/TTS/aws_polly.py index e1d5318..efd762b 100644 --- a/TTS/aws_polly.py +++ b/TTS/aws_polly.py @@ -37,7 +37,7 @@ class AWSPolly: voice = self.randomvoice() else: if not settings.config["settings"]["tts"]["aws_polly_voice"]: - return ValueError( + raise ValueError( f"Please set the TOML variable AWS_VOICE to a valid voice. options are: {voices}" ) voice = str(settings.config["settings"]["tts"]["aws_polly_voice"]).capitalize() @@ -64,10 +64,12 @@ class AWSPolly: sys.exit(-1) except ProfileNotFound: print("You need to install the AWS CLI and configure your profile") - print(""" + print( + """ Linux: https://docs.aws.amazon.com/polly/latest/dg/setup-aws-cli.html Windows: https://docs.aws.amazon.com/polly/latest/dg/install-voice-plugin2.html - """) + """ + ) sys.exit(-1) def randomvoice(self): diff --git a/TTS/streamlabs_polly.py b/TTS/streamlabs_polly.py index bcfa175..8078af5 100644 --- a/TTS/streamlabs_polly.py +++ b/TTS/streamlabs_polly.py @@ -37,7 +37,7 @@ class StreamlabsPolly: voice = self.randomvoice() else: if not settings.config["settings"]["tts"]["streamlabs_polly_voice"]: - return ValueError( + raise ValueError( f"Please set the config variable STREAMLABS_VOICE to a valid voice. options are: {voices}" ) voice = str(settings.config["settings"]["tts"]["streamlabs_polly_voice"]).capitalize() @@ -51,7 +51,7 @@ class StreamlabsPolly: voice_data = requests.get(response.json()["speak_url"]) with open(filepath, "wb") as f: f.write(voice_data.content) - except (KeyError, JSONDecodeError) as e: + except (KeyError, JSONDecodeError): try: if response.json()["error"] == "No text specified!": raise ValueError("Please specify a text to convert to speech.") From 359e9d8c218d547a22e83fe3b23a1654954a23ad Mon Sep 17 00:00:00 2001 From: Jason Date: Tue, 12 Jul 2022 11:50:45 -0400 Subject: [PATCH 124/126] fixed STREAMLABS_POLLY_VOICE typo see https://github.com/elebumm/RedditVideoMakerBot/pull/880/files/1d3410646d674334b2169e0e59b1a731c00f1faf..3068cead0c0e4479b93ce3041cc698c92cba96d3#r915594390 --- TTS/streamlabs_polly.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/TTS/streamlabs_polly.py b/TTS/streamlabs_polly.py index 8078af5..75c4f49 100644 --- a/TTS/streamlabs_polly.py +++ b/TTS/streamlabs_polly.py @@ -38,7 +38,7 @@ class StreamlabsPolly: else: if not settings.config["settings"]["tts"]["streamlabs_polly_voice"]: raise ValueError( - f"Please set the config variable STREAMLABS_VOICE to a valid voice. options are: {voices}" + f"Please set the config variable STREAMLABS_POLLY_VOICE to a valid voice. options are: {voices}" ) voice = str(settings.config["settings"]["tts"]["streamlabs_polly_voice"]).capitalize() body = {"voice": voice, "text": text, "service": "polly"} From 2cf00a1da8813fa42b94261e1196edb331bf3ca4 Mon Sep 17 00:00:00 2001 From: Jason Date: Tue, 12 Jul 2022 11:52:03 -0400 Subject: [PATCH 125/126] chore: updated version and added a branch tag --- main.py | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/main.py b/main.py index b4737e0..9afc15c 100755 --- a/main.py +++ b/main.py @@ -16,7 +16,9 @@ from video_creation.final_video import make_final_video from video_creation.screenshot_downloader import download_screenshots_of_reddit_posts from video_creation.voices import save_text_to_mp3 -VERSION = "2.3" +__VERSION__ = "2.3" +__BRANCH__ = "master" + print( """ ██████╗ ███████╗██████╗ ██████╗ ██╗████████╗ ██╗ ██╗██╗██████╗ ███████╗ ██████╗ ███╗ ███╗ █████╗ ██╗ ██╗███████╗██████╗ From 900816535bd5e4b0291baf5f09533e559ef652db Mon Sep 17 00:00:00 2001 From: Jason Date: Tue, 12 Jul 2022 11:52:52 -0400 Subject: [PATCH 126/126] oh GOD i almost did a 2.2 again. --- main.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/main.py b/main.py index 9afc15c..10ab3c1 100755 --- a/main.py +++ b/main.py @@ -33,7 +33,7 @@ print( print_markdown( "### Thanks for using this tool! [Feel free to contribute to this project on GitHub!](https://lewismenelaws.com) If you have any questions, feel free to reach out to me on Twitter or submit a GitHub issue. You can find solutions to many common problems in the [Documentation](https://luka-hietala.gitbook.io/documentation-for-the-reddit-bot/)" ) -print_step(f"You are using v{VERSION} of the bot") +print_step(f"You are using v{__VERSION__} of the bot") def main(POST_ID=None):