diff --git a/.env.template b/.env.template index 809c6fc..143df0a 100644 --- a/.env.template +++ b/.env.template @@ -20,3 +20,5 @@ OPACITY="1" # see TTSwrapper.py for all valid options VOICE="en_us_001" # e.g. en_us_002 +# IN-PROGRESS - not yet implemented +STORYMODE="False" diff --git a/.github/dependabot.yml b/.github/dependabot.yml new file mode 100644 index 0000000..ba1c6b8 --- /dev/null +++ b/.github/dependabot.yml @@ -0,0 +1,11 @@ +# To get started with Dependabot version updates, you'll need to specify which +# package ecosystems to update and where the package manifests are located. +# Please see the documentation for all configuration options: +# https://docs.github.com/github/administering-a-repository/configuration-options-for-dependency-updates + +version: 2 +updates: + - package-ecosystem: "pip" # See documentation for possible values + directory: "/" # Location of package manifests + schedule: + interval: "daily" diff --git a/.github/workflows/codeql-analysis.yml b/.github/workflows/codeql-analysis.yml new file mode 100644 index 0000000..c04b714 --- /dev/null +++ b/.github/workflows/codeql-analysis.yml @@ -0,0 +1,72 @@ +# For most projects, this workflow file will not need changing; you simply need +# to commit it to your repository. +# +# You may wish to alter this file to override the set of languages analyzed, +# or to provide custom queries or build logic. +# +# ******** NOTE ******** +# We have attempted to detect the languages in your repository. Please check +# the `language` matrix defined below to confirm you have the correct set of +# supported CodeQL languages. +# +name: "CodeQL" + +on: + push: + branches: [ "master" ] + pull_request: + # The branches below must be a subset of the branches above + branches: [ "master" ] + schedule: + - cron: '16 14 * * 3' + +jobs: + analyze: + name: Analyze + runs-on: ubuntu-latest + permissions: + actions: read + contents: read + security-events: write + + strategy: + fail-fast: false + matrix: + language: [ 'python' ] + # CodeQL supports [ 'cpp', 'csharp', 'go', 'java', 'javascript', 'python', 'ruby' ] + # Learn more about CodeQL language support at https://aka.ms/codeql-docs/language-support + + steps: + - name: Checkout repository + uses: actions/checkout@v3 + + # Initializes the CodeQL tools for scanning. + - name: Initialize CodeQL + uses: github/codeql-action/init@v2 + with: + languages: ${{ matrix.language }} + # If you wish to specify custom queries, you can do so here or in a config file. + # By default, queries listed here will override any specified in a config file. + # Prefix the list here with "+" to use these queries and those in the config file. + + # Details on CodeQL's query packs refer to : https://docs.github.com/en/code-security/code-scanning/automatically-scanning-your-code-for-vulnerabilities-and-errors/configuring-code-scanning#using-queries-in-ql-packs + # queries: security-extended,security-and-quality + + + # Autobuild attempts to build any compiled languages (C/C++, C#, or Java). + # If this step fails, then you should remove it and run the build manually (see below) + - name: Autobuild + uses: github/codeql-action/autobuild@v2 + + # ℹ️ Command-line programs to run using the OS shell. + # 📚 See https://docs.github.com/en/actions/using-workflows/workflow-syntax-for-github-actions#jobsjob_idstepsrun + + # If the Autobuild fails above, remove it and uncomment the following three lines. + # modify them (or add more) to build your code if your project, please refer to the EXAMPLE below for guidance. + + # - run: | + # echo "Run, Build Application using script" + # ./location_of_script_within_repo/buildscript.sh + + - name: Perform CodeQL Analysis + uses: github/codeql-action/analyze@v2 diff --git a/README.md b/README.md index 0f7c5a8..5c2d886 100644 --- a/README.md +++ b/README.md @@ -1,16 +1,8 @@ # Reddit Video Maker Bot 🎥 -https://user-images.githubusercontent.com/6053155/170525726-2db23ae0-97b8-4bd1-8c95-00da60ce099f.mp4 - All done WITHOUT video editing or asset compiling. Just pure ✨programming magic✨. -Created by Lewis Menelaws & [TMRRW](https://tmrrwinc.ca) - -[ - - - -](https://tmrrwinc.ca) +Created by Lewis Menelaws & Heavily modified by [Jason Cameron](https://github.com/JasonLovesDoggo) ## Motivation 🤔 @@ -27,7 +19,7 @@ The only original thing being done is the editing and gathering of all materials ## Requirements -- Python 3.6+ +- Python 3.6+ (3.10 is recommended tho ) - Playwright (this should install automatically in installation) ## Installation 👩‍💻 @@ -42,6 +34,10 @@ The only original thing being done is the editing and gathering of all materials 6. Run `python3 main.py` 7. Enjoy 😎 +## Video +https://user-images.githubusercontent.com/19284904/172727335-7e11c816-3484-4033-a535-4b061751fa3b.mp4 + + ## Contributing & Ways to improve 📈 In its current state, this bot does exactly what it needs to do. However, lots of improvements can be made. diff --git a/reddit/subreddit.py b/reddit/subreddit.py index cc5bc02..6a2c273 100644 --- a/reddit/subreddit.py +++ b/reddit/subreddit.py @@ -1,11 +1,10 @@ -import random from os import getenv, environ import praw from utils.console import print_step, print_substep +from utils.subreddit import get_subreddit_undone from utils.videos import check_done -from utils.subreddit import get_hottest_undone TEXT_WHITELIST = set("abcdefghijklmnopqrstuvwxyz ABCDEFGHIJKLMNOPQRSTUVWXYZ 1234567890") @@ -19,7 +18,7 @@ def get_subreddit_threads(): Returns a list of threads from the AskReddit subreddit. """ global submission - print_step("Logging into Reddit.") + print_substep("Logging into Reddit.") content = {} if getenv("REDDIT_2FA").casefold() == "yes": @@ -44,7 +43,9 @@ def get_subreddit_threads(): Ask user for subreddit input """ print_step("Getting subreddit threads...") - if not getenv("SUBREDDIT"): # note to self. you can have multiple subreddits via reddit.subreddit("redditdev+learnpython") + if not getenv( + "SUBREDDIT" + ): # note to self. you can have multiple subreddits via reddit.subreddit("redditdev+learnpython") subreddit = reddit.subreddit( input("What subreddit would you like to pull from? ") ) # if the env isnt set, ask user @@ -60,8 +61,8 @@ def get_subreddit_threads(): submission = reddit.submission(id=getenv("POST_ID")) else: threads = subreddit.hot(limit=25) - submission = get_hottest_undone(threads) - submission = check_done(submission) # double checking + submission = get_subreddit_undone(threads, subreddit) + submission = check_done(submission) # double checking if submission is None: return get_subreddit_threads() # submission already done. rerun upvotes = submission.score @@ -79,9 +80,9 @@ def get_subreddit_threads(): ) # todo use global instend of env vars environ["VIDEO_ID"] = str(textify(submission.id)) try: - content["thread_url"] = submission.url content["thread_title"] = submission.title + # ontent["thread_content"] = submission.content content["comments"] = [] for top_level_comment in submission.comments: diff --git a/requirements.txt b/requirements.txt index 41bf400..0173901 100644 --- a/requirements.txt +++ b/requirements.txt @@ -7,4 +7,3 @@ python-dotenv==0.20.0 typed-ast~=1.5.4 requests~=2.27.1 pytube~=12.1.0 -typing diff --git a/utils/subreddit.py b/utils/subreddit.py index 4b82f2c..3fc1613 100644 --- a/utils/subreddit.py +++ b/utils/subreddit.py @@ -1,6 +1,7 @@ from typing import List import json + def get_subreddit_undone(submissions: List, subreddit): """ recursively checks if the top submission in the list was already done. @@ -11,7 +12,10 @@ def get_subreddit_undone(submissions: List, subreddit): if already_done(done_videos, submission): continue return submission - return get_subreddit_undone(subreddit.top(time_filter="hour"), subreddit) # all of the videos in hot have already been done + return get_subreddit_undone( + subreddit.top(time_filter="hour"), subreddit + ) # all of the videos in hot have already been done + def already_done(done_videos: list, submission): diff --git a/utils/videos.py b/utils/videos.py index 51a2704..a7b2012 100755 --- a/utils/videos.py +++ b/utils/videos.py @@ -4,9 +4,7 @@ from os import getenv from utils.console import print_step -def check_done( - redditobj, -): # don't set this to be run anyplace that isn't subreddit.py bc of inspect stack +def check_done(redditobj): # don't set this to be run anyplace that isn't subreddit.py bc of inspect stack """params: reddit_object: The Reddit Object you received in askreddit.py""" with open("./video_creation/data/videos.json", "r") as done_vids_raw: diff --git a/video_creation/TTSwrapper.py b/video_creation/TTSwrapper.py index 2ebdb28..3854c46 100644 --- a/video_creation/TTSwrapper.py +++ b/video_creation/TTSwrapper.py @@ -5,6 +5,8 @@ import re import sox import requests +from moviepy.audio.AudioClip import concatenate_audioclips, CompositeAudioClip +from moviepy.audio.io.AudioFileClip import AudioFileClip from requests.adapters import HTTPAdapter, Retry # from profanity_filter import ProfanityFilter @@ -67,11 +69,11 @@ class TTTTSWrapper: # TikTok Text-to-Speech Wrapper self.URI_BASE = "https://api16-normal-useast5.us.tiktokv.com/media/api/text/speech/invoke/?text_speaker=" def tts( - self, - req_text: str = "TikTok Text To Speech", - filename: str = "title.mp3", - random_speaker: bool = False, - censer=False, + self, + req_text: str = "TikTok Text To Speech", + filename: str = "title.mp3", + random_speaker: bool = False, + censer=False, ): req_text = req_text.replace("+", "plus").replace(" ", "+").replace("&", "and") if censer: @@ -89,6 +91,7 @@ class TTTTSWrapper: # TikTok Text-to-Speech Wrapper audio_clips = [] cbn = sox.Combiner() + cbn.set_input_format(file_type=['mp3']) chunkId = 0 for chunk in chunks: @@ -115,14 +118,20 @@ class TTTTSWrapper: # TikTok Text-to-Speech Wrapper audio_clips.append(filename.replace(".mp3", f"-{chunkId}.mp3")) chunkId = chunkId + 1 - - if(len(audio_clips) > 1): - cbn.convert(samplerate=44100, n_channels=2) - cbn.build( - audio_clips, filename, 'concatenate' - ) - else: - os.rename(audio_clips[0], filename) + try: + if len(audio_clips) > 1: + cbn.convert(samplerate=44100, n_channels=2) + cbn.build(audio_clips, filename, "concatenate") + else: + os.rename(audio_clips[0], filename) + except sox.core.SoxError: + for clip in audio_clips: + i = audio_clips.index(clip) # get the index of the clip + audio_clips = audio_clips[:i] + [AudioFileClip(clip)] + audio_clips[ + i + 1:] # replace the clip with an AudioFileClip + audio_concat = concatenate_audioclips(audio_clips) + audio_composite = CompositeAudioClip([audio_concat]) + audio_composite.write_audiofile(filename, 44100, 2, 2000, None) @staticmethod def randomvoice(): diff --git a/video_creation/screenshot_downloader.py b/video_creation/screenshot_downloader.py index fda30f4..89d6728 100644 --- a/video_creation/screenshot_downloader.py +++ b/video_creation/screenshot_downloader.py @@ -2,12 +2,14 @@ import json from os import getenv from pathlib import Path -from playwright.async_api import async_playwright +# from playwright.async_api import async_playwright from playwright.sync_api import sync_playwright, ViewportSize from rich.progress import track from utils.console import print_step, print_substep +storymode = False + def download_screenshots_of_reddit_posts(reddit_object, screenshot_num): """Downloads screenshots of reddit posts as they are seen on the web. @@ -32,7 +34,7 @@ def download_screenshots_of_reddit_posts(reddit_object, screenshot_num): else: cookie_file = open("./video_creation/data/cookie-light-mode.json") cookies = json.load(cookie_file) - context.add_cookies(cookies) + context.add_cookies(cookies) # load preference cookies # Get the thread screenshot page = context.new_page() page.set_viewport_size(ViewportSize(width=1920, height=1080)) @@ -52,20 +54,24 @@ def download_screenshots_of_reddit_posts(reddit_object, screenshot_num): page.locator('[data-test-id="post-content"]').screenshot( path="assets/temp/png/title.png" ) + if storymode: + page.locator('[data-click-id="text"]').screenshot( + path="assets/temp/png/story_content.png" + ) + else: + for idx, comment in track( + enumerate(reddit_object["comments"]), "Downloading screenshots..." + ): - for idx, comment in track( - enumerate(reddit_object["comments"]), "Downloading screenshots..." - ): - - # Stop if we have reached the screenshot_num - if idx >= screenshot_num: - break + # Stop if we have reached the screenshot_num + if idx >= screenshot_num: + break - if page.locator('[data-testid="content-gate"]').is_visible(): - page.locator('[data-testid="content-gate"] button').click() + if page.locator('[data-testid="content-gate"]').is_visible(): + page.locator('[data-testid="content-gate"] button').click() - page.goto(f'https://reddit.com{comment["comment_url"]}') - page.locator(f"#t1_{comment['comment_id']}").screenshot( - path=f"assets/temp/png/comment_{idx}.png" - ) + page.goto(f'https://reddit.com{comment["comment_url"]}') + page.locator(f"#t1_{comment['comment_id']}").screenshot( + path=f"assets/temp/png/comment_{idx}.png" + ) print_substep("Screenshots downloaded Successfully.", style="bold green") diff --git a/video_creation/voices.py b/video_creation/voices.py index 32f468e..be7bc7d 100644 --- a/video_creation/voices.py +++ b/video_creation/voices.py @@ -1,3 +1,4 @@ +from os import getenv from pathlib import Path import sox @@ -34,6 +35,13 @@ def save_text_to_mp3(reddit_obj): length += MP3(f"assets/temp/mp3/title.mp3").info.length except HeaderNotFoundError: # note to self AudioFileClip length += sox.file_info.duration(f"assets/temp/mp3/title.mp3") + if getenv("STORYMODE").casefold() == "true": + ttttsw.tts( + sanitize_text(reddit_obj["thread_content"]), + filename=f"assets/temp/mp3/story_content.mp3", + random_speaker=False, + ) + #'story_content' com = 0 for comment in track((reddit_obj["comments"]), "Saving..."): # ! Stop creating mp3 files if the length is greater than VIDEO_LENGTH seconds. This can be longer, but this is just a good_voices starting point @@ -62,5 +70,5 @@ def save_text_to_mp3(reddit_obj): # remove(f"assets/temp/png/comment_{com}.png")# todo might cause odd un-syncing print_substep("Saved Text to MP3 files Successfully.", style="bold green") - # ! Return the index so we know how many screenshots of comments we need to make. + # ! Return the index, so we know how many screenshots of comments we need to make. return length, com