diff --git a/.pylintrc b/.pylintrc index b03c808..9bb7919 100644 --- a/.pylintrc +++ b/.pylintrc @@ -60,7 +60,7 @@ ignored-modules= # Python code to execute, usually for sys.path manipulation such as # pygtk.require(). -#init-hook= +init-hook='import sys; sys.path.append("/")' # Use multiple processes to speed up Pylint. Specifying 0 will auto-detect the # number of processors available to use. diff --git a/README.md b/README.md index 8aa310d..ea3a187 100644 --- a/README.md +++ b/README.md @@ -32,7 +32,7 @@ The only original thing being done is the editing and gathering of all materials ## Requirements -- Python 3.6+ +- Python 3.7+ - Playwright (this should install automatically in installation) ## Installation 👩‍💻 diff --git a/main.py b/main.py index 7f5de77..326654a 100755 --- a/main.py +++ b/main.py @@ -1,18 +1,18 @@ #!/usr/bin/env python from subprocess import Popen -from dotenv import load_dotenv from os import getenv, name +from dotenv import load_dotenv from reddit.subreddit import get_subreddit_threads from utils.cleanup import cleanup from utils.console import print_markdown, print_step +from utils.checker import check_env # from utils.checker import envUpdate from video_creation.background import download_background, chop_background_video from video_creation.final_video import make_final_video from video_creation.screenshot_downloader import download_screenshots_of_reddit_posts from video_creation.voices import save_text_to_mp3 -from utils.checker import check_env VERSION = 2.1 print( @@ -43,18 +43,23 @@ def main( load_dotenv() cleanup() + reddit_object = get_subreddit_threads( subreddit_, thread_link_, number_of_comments ) - length, number_of_comments = save_text_to_mp3(reddit_object) + + + length, number_of_comments = save_text_to_mp3(reddit_object) download_screenshots_of_reddit_posts(reddit_object, number_of_comments) download_background(background) chop_background_video(length) + make_final_video(number_of_comments, length, filename) + def run_many(times): for x in range(1, times + 1): print_step( diff --git a/reddit/subreddit.py b/reddit/subreddit.py index bb29610..5270ae7 100644 --- a/reddit/subreddit.py +++ b/reddit/subreddit.py @@ -8,13 +8,15 @@ from prawcore.exceptions import ( import random import os import re -from os import getenv, environ +from os import getenv import praw +from praw.models import MoreComments from utils.console import print_step, print_substep from utils.subreddit import get_subreddit_undone from utils.videos import check_done + from praw.models import MoreComments TEXT_WHITELIST = set( @@ -32,6 +34,7 @@ def try_env(param, backup): return backup + def get_subreddit_threads(subreddit_, thread_link_, number_of_comments): """ Takes subreddit_ as parameter which defaults to None, but in this @@ -44,6 +47,7 @@ def get_subreddit_threads(subreddit_, thread_link_, number_of_comments): global submission load_dotenv() + if os.getenv("REDDIT_2FA", default="no").casefold() == "yes": print( "\nEnter your two-factor authentication code from your authenticator app.\n", end=" " @@ -52,6 +56,7 @@ def get_subreddit_threads(subreddit_, thread_link_, number_of_comments): pw = os.getenv("REDDIT_PASSWORD") passkey = f"{pw}:{code}" else: + passkey = os.getenv("REDDIT_PASSWORD") content = {} @@ -126,6 +131,7 @@ def get_subreddit_threads(subreddit_, thread_link_, number_of_comments): ratio = submission.upvote_ratio * 100 num_comments = submission.num_comments + print_substep( f"[bold]Video will be: [cyan]{submission.title}[/cyan] :thumbsup:\n" + f"[blue] Thread has {upvotes} and upvote ratio of {ratio}%\n" @@ -158,16 +164,20 @@ def get_subreddit_threads(subreddit_, thread_link_, number_of_comments): print_substep("AskReddit threads retrieved successfully.", style="bold green") + content["thread_url"] = f"https://reddit.com{submission.permalink}" content["thread_title"] = submission.title content["thread_post"] = submission.selftext + content["thread_id"] = submission.id content["comments"] = [] + for top_level_comment in submission.comments: if isinstance(top_level_comment, MoreComments): continue if top_level_comment.body in ["[removed]", "[deleted]"]: continue # # see https://github.com/JasonLovesDoggo/RedditVideoMakerBot/issues/78 if not top_level_comment.stickied: + if len(top_level_comment.body) <= int(try_env("MAX_COMMENT_LENGTH", 500)): content["comments"].append( { @@ -178,4 +188,5 @@ def get_subreddit_threads(subreddit_, thread_link_, number_of_comments): ) print_substep("Received subreddit threads Successfully.", style="bold green") + return content diff --git a/setup.py b/setup.py old mode 100755 new mode 100644 index a2234e7..63992e7 --- a/setup.py +++ b/setup.py @@ -5,11 +5,10 @@ # Imports import os import subprocess -import re -from utils.console import print_markdown -from utils.console import print_step from rich.console import Console from utils.loader import Loader +from utils.console import print_markdown +from utils.console import print_step from utils.console import handle_input console = Console() @@ -61,6 +60,7 @@ console.print("[bold green]Reddit 2FA (yes or no)") console.print("[bold green]Opacity (range of 0-1, decimals are OK)") console.print("[bold green]Subreddit (without r/ or /r/)") console.print("[bold green]Theme (light or dark)") +console.print("[bold green]Random Thread (yes or no)") console.print( "[green]If you don't have these, please follow the instructions in the README.md file to set them up." ) @@ -140,23 +140,34 @@ theme = handle_input( r"(light)|(dark)", "You need to input 'light' or 'dark'", ) +Random_thread = handle_input( + "Random Thread? (yes/no)", + False, + r"(yes)|(no)", + "You need to input either yes or no", +) + loader = Loader("Attempting to save your credentials...", "Done!").start() # you can also put a while loop here, e.g. while VideoIsBeingMade == True: ... console.print("Writing to the .env file...") -with open(".env", "w") as f: +with open(".env", "w", encoding="utf-8") as f: f.write( f"""REDDIT_CLIENT_ID="{client_id}" REDDIT_CLIENT_SECRET="{client_sec}" REDDIT_USERNAME="{user}" REDDIT_PASSWORD="{passw}" REDDIT_2FA="{twofactor}" +RANDOM_THREAD="{Random_thread}" THEME="{theme}" SUBREDDIT="{subreddit}" OPACITY={opacity} +VOICE="Matthew" +TTsChoice="polly" +STORYMODE="False" """ ) -with open(".setup-done-before", "w") as f: +with open(".setup-done-before", "w", encoding="utf-8") as f: f.write( "This file blocks the setup assistant from running again. Delete this file to run setup again." ) diff --git a/utils/checker.py b/utils/checker.py index efc3e6b..acf2536 100755 --- a/utils/checker.py +++ b/utils/checker.py @@ -22,10 +22,10 @@ def check_env() -> bool: return True if not os.path.exists(".env"): console.print("[red]Couldn't find the .env file, creating one now.") - with open(".env", "x") as file: + with open(".env", "x", encoding="utf-8") as file: file.write("") success = True - with open(".env.template", "r") as template: + with open(".env.template", "r", encoding="utf-8") as template: # req_envs = [env.split("=")[0] for env in template.readlines() if "=" in env] matching = {} explanations = {} @@ -36,7 +36,11 @@ def check_env() -> bool: req_envs = [] var_optional = False for line in template.readlines(): - if line.startswith("#") is not True and "=" in line and var_optional is not True: + if ( + line.startswith("#") is not True + and "=" in line + and var_optional is not True + ): req_envs.append(line.split("=")[0]) if "#" in line: examples[line.split("=")[0]] = "#".join( @@ -60,8 +64,10 @@ def check_env() -> bool: ) var_optional = False elif line.startswith("#MATCH_TYPE "): + types[req_envs[-1] ] = eval(line.removeprefix("#MATCH_TYPE ")[:-1].split()[0]) + var_optional = False elif line.startswith("#EXPLANATION "): explanations[req_envs[-1] @@ -88,9 +94,9 @@ def check_env() -> bool: try: temp = types[env](value) if env in bounds.keys(): - (bounds[env][0] <= temp or incorrect.add(env)) and len(bounds[env]) > 1 and ( - bounds[env][1] >= temp or incorrect.add(env) - ) + (bounds[env][0] <= temp or incorrect.add(env)) and len( + bounds[env] + ) > 1 and (bounds[env][1] >= temp or incorrect.add(env)) except ValueError: incorrect.add(env) @@ -116,13 +122,17 @@ def check_env() -> bool: for env in missing: table.add_row( env, + explanations[env] if env in explanations.keys( ) else "No explanation given", examples[env] if env in examples.keys() else "", str(bounds[env][0]) if env in bounds.keys( ) and bounds[env][1] is not None else "", + str(bounds[env][1]) - if env in bounds.keys() and len(bounds[env]) > 1 and bounds[env][1] is not None + if env in bounds.keys() + and len(bounds[env]) > 1 + and bounds[env][1] is not None else "", ) console.print(table) @@ -138,6 +148,7 @@ def check_env() -> bool: title_justify="left", title_style="#C0CAF5 bold", ) + table.add_column("Variable", justify="left", style="#7AA2F7 bold", no_wrap=True) table.add_column("Current value", justify="left", @@ -146,18 +157,21 @@ def check_env() -> bool: style="#BB9AF7", no_wrap=False) table.add_column("Example", justify="center", style="#F7768E", no_wrap=True) + table.add_column("Min", justify="right", style="#F7768E", no_wrap=True) table.add_column("Max", justify="left", style="#F7768E", no_wrap=True) for env in incorrect: table.add_row( env, os.getenv(env), + explanations[env] if env in explanations.keys( ) else "No explanation given", str(types[env].__name__) if env in types.keys() else "str", str(bounds[env][0]) if env in bounds.keys() else "None", str(bounds[env][1]) if env in bounds.keys() and len( bounds[env]) > 1 else "None", + ) missing.add(env) console.print(table) @@ -171,7 +185,7 @@ def check_env() -> bool: console.print("[red]Aborting: Unresolved missing variables") return False if len(incorrect): - with open(".env", "r+") as env_file: + with open(".env", "r+", encoding="utf-8") as env_file: lines = [] for line in env_file.readlines(): line.split("=")[0].strip( @@ -179,9 +193,10 @@ def check_env() -> bool: env_file.seek(0) env_file.write("\n".join(lines)) env_file.truncate() - console.print( - "[green]Successfully removed incorrectly set variables from .env") - with open(".env", "a") as env_file: + + console.print("[green]Successfully removed incorrectly set variables from .env") + with open(".env", "a", encoding="utf-8") as env_file: + for env in missing: env_file.write( env @@ -196,6 +211,7 @@ def check_env() -> bool: if env in explanations.keys() else "Incorrect input. Try again.", bounds[env][0] if env in bounds.keys() else None, + bounds[env][1] if env in bounds.keys() and len( bounds[env]) > 1 else None, oob_errors[env] if env in oob_errors.keys( @@ -204,6 +220,7 @@ def check_env() -> bool: + ( explanations[env] if env in explanations.keys( ) else "No info available" + ), ) ) diff --git a/utils/cleanup.py b/utils/cleanup.py index 558307d..c8cb9f1 100644 --- a/utils/cleanup.py +++ b/utils/cleanup.py @@ -12,6 +12,7 @@ def cleanup() -> int: count = 0 files = [f for f in os.listdir(".") if f.endswith( ".mp4") and "temp" in f.lower()] + count += len(files) for f in files: os.remove(f) diff --git a/utils/console.py b/utils/console.py index a0b9117..f217a85 100644 --- a/utils/console.py +++ b/utils/console.py @@ -63,10 +63,14 @@ def handle_input( except ValueError: console.print("[red]" + err_message) # Type conversion failed continue - if nmin is not None and len(user_input) < nmin: # Check if string is long enough + if ( + nmin is not None and len(user_input) < nmin + ): # Check if string is long enough console.print("[red]" + oob_error) continue - if nmax is not None and len(user_input) > nmax: # Check if string is not too long + if ( + nmax is not None and len(user_input) > nmax + ): # Check if string is not too long console.print("[red]" + oob_error) continue break diff --git a/utils/subreddit.py b/utils/subreddit.py index e350722..4151137 100644 --- a/utils/subreddit.py +++ b/utils/subreddit.py @@ -1,4 +1,3 @@ -from typing import List import json from os import getenv from utils.console import print_substep @@ -9,15 +8,16 @@ def get_subreddit_undone(submissions: list, subreddit): Args: submissions (list): List of posts that are going to potentially be generated into a video - subreddit (praw.Reddit.SubredditHelper): Chosen subreddit + subreddit (praw.Reddit.SubredditHelper): Chosen subreddit Returns: Any: The submission that has not been done """ - """ - recursively checks if the top submission in the list was already done. - """ - with open("./video_creation/data/videos.json", "r") as done_vids_raw: + # recursively checks if the top submission in the list was already done. + + with open( + "./video_creation/data/videos.json", "r", encoding="utf-8" + ) as done_vids_raw: done_videos = json.load(done_vids_raw) for submission in submissions: if already_done(done_videos, submission): @@ -40,6 +40,7 @@ def get_subreddit_undone(submissions: list, subreddit): def already_done(done_videos: list, submission) -> bool: """Checks to see if the given submission is in the list of videos + Args: done_videos (list): Finished videos submission (Any): The submission diff --git a/utils/videos.py b/utils/videos.py index 27f26bc..dcf5a2e 100755 --- a/utils/videos.py +++ b/utils/videos.py @@ -1,4 +1,5 @@ import json +from typing import Union from os import getenv from utils.console import print_step @@ -6,7 +7,9 @@ from utils.console import print_step def check_done( redditobj: dict[str], -) -> dict[str] | None: # don't set this to be run anyplace that isn't subreddit.py bc of inspect stack + +) -> Union[dict[str], None]: + # don't set this to be run anyplace that isn't subreddit.py bc of inspect stack """Checks if the chosen post has already been generated Args: @@ -16,7 +19,9 @@ def check_done( dict[str]|None: Reddit object in args """ - with open("./video_creation/data/videos.json", "r") as done_vids_raw: + with open( + "./video_creation/data/videos.json", "r", encoding="utf-8" + ) as done_vids_raw: done_videos = json.load(done_vids_raw) for video in done_videos: if video["id"] == str(redditobj): diff --git a/utils/voice.py b/utils/voice.py index e78ddad..c4f27bf 100644 --- a/utils/voice.py +++ b/utils/voice.py @@ -2,7 +2,7 @@ import re def sanitize_text(text: str) -> str: - """Sanitizes the text for tts. + r"""Sanitizes the text for tts. What gets removed: - following characters`^_~@!&;#:-%“”‘"%*/{}[]()\|<>?=+` - any http or https links diff --git a/video_creation/background.py b/video_creation/background.py index 26795ed..85c7cbf 100644 --- a/video_creation/background.py +++ b/video_creation/background.py @@ -9,6 +9,10 @@ import os from random import randrange from moviepy.video.io.ffmpeg_tools import ffmpeg_extract_subclip from moviepy.editor import VideoFileClip +from moviepy.editor import VideoFileClip +from moviepy.video.io.ffmpeg_tools import ffmpeg_extract_subclip +from pytube import YouTube + from utils.console import print_step, print_substep @@ -90,12 +94,18 @@ def chop_background_video(video_length: int): background = VideoFileClip(f"assets/backgrounds/{choice}") - start_time, end_time = get_start_and_end_times( - video_length, background.duration) - ffmpeg_extract_subclip( - f"assets/backgrounds/{choice}", - start_time, - end_time, - targetname="assets/temp/background.mp4", - ) + + start_time, end_time = get_start_and_end_times(video_length, background.duration) + try: + ffmpeg_extract_subclip( + f"assets/backgrounds/{choice}", + start_time, + end_time, + targetname="assets/temp/background.mp4", + ) + except (OSError, IOError): # ffmpeg issue see #348 + print_substep("FFMPEG issue. Trying again...") + with VideoFileClip(f"assets/backgrounds/{choice}") as video: + new = video.subclip(start_time, end_time) + new.write_videofile("assets/temp/background.mp4") print_substep("Background video chopped successfully!", style="bold green") diff --git a/video_creation/final_video.py b/video_creation/final_video.py index 667fe21..9d23ca2 100755 --- a/video_creation/final_video.py +++ b/video_creation/final_video.py @@ -1,7 +1,9 @@ #!/usr/bin/env python3 import json -import os import time +import multiprocessing +import re +import os from os.path import exists import os @@ -19,7 +21,6 @@ from moviepy.editor import ( from moviepy.video.io import ffmpeg_tools from rich.console import Console -from reddit import subreddit from utils.cleanup import cleanup from utils.console import print_step, print_substep @@ -28,7 +29,9 @@ console = Console() W, H = 1080, 1920 -def make_final_video(number_of_clips: int, length: int, final_vid_path: str): + +def make_final_video(number_of_clips: int, length: int, final_vid_path: str, reddit_obj: dict[str]): + """Gathers audio clips, gathers all screenshots, stitches them together and saves the final video to assets/temp Args: @@ -139,6 +142,11 @@ def make_final_video(number_of_clips: int, length: int, final_vid_path: str): ) image_concat.audio = audio_composite final = CompositeVideoClip([background_clip, image_concat]) + title = re.sub(r"[^\w\s-]", "", reddit_obj["thread_title"]) + idx = re.sub(r"[^\w\s-]", "", reddit_obj["thread_id"]) + filename = f"{title}.mp4" + subreddit = os.getenv("SUBREDDIT"); + if final_vid_path is None: final_vid_path = re.sub( @@ -154,15 +162,19 @@ def make_final_video(number_of_clips: int, length: int, final_vid_path: str): print_substep("the results folder didn't exist so I made it") os.mkdir("./results") + final.write_videofile( "assets/temp/temp.mp4", fps=30, audio_codec="aac", audio_bitrate="192k", verbose=False, + threads=multiprocessing.cpu_count(), ) ffmpeg_tools.ffmpeg_extract_subclip( + "assets/temp/temp.mp4", 0, length, targetname=f"results/{final_vid_path}" + ) # os.remove("assets/temp/temp.mp4") @@ -172,25 +184,29 @@ def make_final_video(number_of_clips: int, length: int, final_vid_path: str): print_substep("See result in the results folder!") print_step( - f"Reddit title: {os.getenv('VIDEO_TITLE')} \n Background Credit: {os.getenv('background_credit')}" + f'Reddit title: { reddit_obj["thread_title"] } \n Background Credit: {os.getenv("background_credit")}' ) + def save_data(filename: str): + """Saves the videos that have already been generated to a JSON file in video_creation/data/videos.json Args: filename (str): The finished video title name """ - with open("./video_creation/data/videos.json", "r+") as raw_vids: + + with open("./video_creation/data/videos.json", "r+", encoding="utf-8") as raw_vids: + done_vids = json.load(raw_vids) - if str(subreddit.submission.id) in [video["id"] for video in done_vids]: + if reddit_id in [video["id"] for video in done_vids]: return # video already done but was specified to continue anyway in the .env file payload = { - "id": str(os.getenv("VIDEO_ID")), + "id": reddit_id, "time": str(int(time.time())), "background_credit": str(os.getenv("background_credit")), - "reddit_title": str(os.getenv("VIDEO_TITLE")), + "reddit_title": reddit_title, "filename": filename, } done_vids.append(payload) @@ -198,6 +214,7 @@ def save_data(filename: str): json.dump(done_vids, raw_vids, ensure_ascii=False, indent=4) + def get_video_title() -> str: """Gets video title from env variable or gives it the name "final_video" @@ -209,3 +226,4 @@ def get_video_title() -> str: return title else: return title[0:30] + "..." + diff --git a/video_creation/screenshot_downloader.py b/video_creation/screenshot_downloader.py index 8d201bb..e3d96b2 100644 --- a/video_creation/screenshot_downloader.py +++ b/video_creation/screenshot_downloader.py @@ -1,11 +1,10 @@ import json -from os import getenv import os +from os import getenv from pathlib import Path -from playwright.async_api import async_playwright -from playwright.sync_api import sync_playwright, ViewportSize -from rich.progress import track +from playwright.async_api import async_playwright # pylint: disable=unused-impor +# do not remove the above line from utils.console import print_step, print_substep import json @@ -15,9 +14,13 @@ from playwright.sync_api import sync_playwright, ViewportSize from rich.progress import track from rich.console import Console + + +from playwright.sync_api import sync_playwright, ViewportSize +from rich.progress import track import translators as ts -console = Console() +from utils.console import print_step, print_substep storymode = False @@ -41,9 +44,13 @@ def download_screenshots_of_reddit_posts(reddit_object: dict[str], screenshot_nu context = browser.new_context() if getenv("THEME").upper() == "DARK": - cookie_file = open("./video_creation/data/cookie-dark-mode.json") + cookie_file = open( + "./video_creation/data/cookie-dark-mode.json", encoding="utf-8" + ) else: - cookie_file = open("./video_creation/data/cookie-light-mode.json") + cookie_file = open( + "./video_creation/data/cookie-light-mode.json", encoding="utf-8" + ) cookies = json.load(cookie_file) context.add_cookies(cookies) # load preference cookies # Get the thread screenshot @@ -64,10 +71,14 @@ def download_screenshots_of_reddit_posts(reddit_object: dict[str], screenshot_nu if getenv("POSTLANG"): print_substep("Translating post...") texts_in_tl = ts.google( - reddit_object["thread_title"], to_language=os.getenv("POSTLANG")) + + reddit_object["thread_title"], to_language=os.getenv("POSTLANG") + ) + page.evaluate( - 'tl_content => document.querySelector(\'[data-test-id="post-content"] > div:nth-child(3) > div > div\').textContent = tl_content', texts_in_tl + "tl_content => document.querySelector('[data-test-id=\"post-content\"] > div:nth-child(3) > div > div').textContent = tl_content", + texts_in_tl, ) else: print_substep("Skipping translation...") @@ -99,10 +110,12 @@ def download_screenshots_of_reddit_posts(reddit_object: dict[str], screenshot_nu if getenv("POSTLANG"): comment_tl = ts.google( - comment["comment_body"], to_language=os.getenv("POSTLANG")) + + comment["comment_body"], to_language=os.getenv("POSTLANG") + ) page.evaluate( - '([tl_content, tl_id]) => document.querySelector(`#t1_${tl_id} > div:nth-child(2) > div > div[data-testid="comment"] > div`).textContent = tl_content', [ - comment_tl, comment['comment_id']] + '([tl_content, tl_id]) => document.querySelector(`#t1_${tl_id} > div:nth-child(2) > div > div[data-testid="comment"] > div`).textContent = tl_content', + [comment_tl, comment["comment_id"]], ) page.locator(f"#t1_{comment['comment_id']}").screenshot(