From f3d8af5a3a6834e7ace554d451d2e0298ac343dd Mon Sep 17 00:00:00 2001 From: PatatjeMC Date: Tue, 7 Jun 2022 20:50:32 +0200 Subject: [PATCH 1/6] Block stickied comments from being used this blocks stickied comments from being used because this is mostly used for bot messages and not about the actual thread. --- reddit/subreddit.py | 9 +++++---- 1 file changed, 5 insertions(+), 4 deletions(-) diff --git a/reddit/subreddit.py b/reddit/subreddit.py index c7ac06d..d8c68bf 100755 --- a/reddit/subreddit.py +++ b/reddit/subreddit.py @@ -58,10 +58,11 @@ def get_subreddit_threads(): content["comments"] = [] for top_level_comment in submission.comments: - if len(top_level_comment.body) <= int(environ["MAX_COMMENT_LENGTH"]): - content["comments"].append( - {"comment_body": top_level_comment.body, "comment_url": top_level_comment.permalink, - "comment_id": top_level_comment.id, }) + if not top_level_comment.stickied: + if len(top_level_comment.body) <= int(environ["MAX_COMMENT_LENGTH"]): + content["comments"].append( + {"comment_body": top_level_comment.body, "comment_url": top_level_comment.permalink, + "comment_id": top_level_comment.id, }) except AttributeError as e: pass From 78036c5ab2e380024c5721be169e9b6991042b78 Mon Sep 17 00:00:00 2001 From: PatatjeMC Date: Tue, 7 Jun 2022 20:55:56 +0200 Subject: [PATCH 2/6] Revert "Block stickied comments from being used" This reverts commit f3d8af5a3a6834e7ace554d451d2e0298ac343dd. --- reddit/subreddit.py | 9 ++++----- 1 file changed, 4 insertions(+), 5 deletions(-) diff --git a/reddit/subreddit.py b/reddit/subreddit.py index d8c68bf..c7ac06d 100755 --- a/reddit/subreddit.py +++ b/reddit/subreddit.py @@ -58,11 +58,10 @@ def get_subreddit_threads(): content["comments"] = [] for top_level_comment in submission.comments: - if not top_level_comment.stickied: - if len(top_level_comment.body) <= int(environ["MAX_COMMENT_LENGTH"]): - content["comments"].append( - {"comment_body": top_level_comment.body, "comment_url": top_level_comment.permalink, - "comment_id": top_level_comment.id, }) + if len(top_level_comment.body) <= int(environ["MAX_COMMENT_LENGTH"]): + content["comments"].append( + {"comment_body": top_level_comment.body, "comment_url": top_level_comment.permalink, + "comment_id": top_level_comment.id, }) except AttributeError as e: pass From 9d2c2cd88f58c348464187f3fe12799c3956cb1d Mon Sep 17 00:00:00 2001 From: PatatjeMC Date: Wed, 8 Jun 2022 20:30:17 +0200 Subject: [PATCH 3/6] Use sox for combining audio in TTS chunk combiner I did this because moviepy has a lot of glitches in the audio. --- video_creation/TTSwrapper.py | 19 +++++++++++++------ 1 file changed, 13 insertions(+), 6 deletions(-) diff --git a/video_creation/TTSwrapper.py b/video_creation/TTSwrapper.py index f5ae5e5..76a5a03 100644 --- a/video_creation/TTSwrapper.py +++ b/video_creation/TTSwrapper.py @@ -1,10 +1,11 @@ import re import base64 +import shutil import os import random import requests -from moviepy.editor import AudioFileClip, concatenate_audioclips, CompositeAudioClip +import sox #from profanity_filter import ProfanityFilter #pf = ProfanityFilter() # Code by @JasonLovesDoggo @@ -75,6 +76,7 @@ class TTTTSWrapper: # TikTok Text-to-Speech Wrapper chunks = [m.group().strip() for m in re.finditer(r' *((.{0,200})(\.|.$))', req_text)] audio_clips = [] + cbn = sox.Combiner() chunkId = 0 for chunk in chunks: @@ -82,16 +84,21 @@ class TTTTSWrapper: # TikTok Text-to-Speech Wrapper vstr = [r.json()["data"]["v_str"]][0] b64d = base64.b64decode(vstr) - with open(f"{filename}-{chunkId}", "wb") as out: + with open(filename.replace(".mp3", f"-{chunkId}.mp3"), "wb") as out: out.write(b64d) - audio_clips.append(AudioFileClip(f"{filename}-{chunkId}")) + audio_clips.append(filename.replace(".mp3", f"-{chunkId}.mp3")) chunkId = chunkId + 1 - audio_concat = concatenate_audioclips(audio_clips) - audio_composite = CompositeAudioClip([audio_concat]) - audio_composite.write_audiofile(filename, 44100, 2, 2000, None) + if(len(audio_clips) > 1): + cbn.convert(samplerate=44100, n_channels=2) + cbn.build( + audio_clips, filename, 'concatenate' + ) + else: + os.rename(audio_clips[0], filename) + @staticmethod def randomvoice(): From 73907026a72d37508624ca458fdaa687ea9abf84 Mon Sep 17 00:00:00 2001 From: PatatjeMC Date: Wed, 8 Jun 2022 20:31:25 +0200 Subject: [PATCH 4/6] Revert "Use sox for combining audio in TTS chunk combiner" This reverts commit 9d2c2cd88f58c348464187f3fe12799c3956cb1d. --- video_creation/TTSwrapper.py | 19 ++++++------------- 1 file changed, 6 insertions(+), 13 deletions(-) diff --git a/video_creation/TTSwrapper.py b/video_creation/TTSwrapper.py index 76a5a03..f5ae5e5 100644 --- a/video_creation/TTSwrapper.py +++ b/video_creation/TTSwrapper.py @@ -1,11 +1,10 @@ import re import base64 -import shutil import os import random import requests -import sox +from moviepy.editor import AudioFileClip, concatenate_audioclips, CompositeAudioClip #from profanity_filter import ProfanityFilter #pf = ProfanityFilter() # Code by @JasonLovesDoggo @@ -76,7 +75,6 @@ class TTTTSWrapper: # TikTok Text-to-Speech Wrapper chunks = [m.group().strip() for m in re.finditer(r' *((.{0,200})(\.|.$))', req_text)] audio_clips = [] - cbn = sox.Combiner() chunkId = 0 for chunk in chunks: @@ -84,21 +82,16 @@ class TTTTSWrapper: # TikTok Text-to-Speech Wrapper vstr = [r.json()["data"]["v_str"]][0] b64d = base64.b64decode(vstr) - with open(filename.replace(".mp3", f"-{chunkId}.mp3"), "wb") as out: + with open(f"{filename}-{chunkId}", "wb") as out: out.write(b64d) - audio_clips.append(filename.replace(".mp3", f"-{chunkId}.mp3")) + audio_clips.append(AudioFileClip(f"{filename}-{chunkId}")) chunkId = chunkId + 1 - if(len(audio_clips) > 1): - cbn.convert(samplerate=44100, n_channels=2) - cbn.build( - audio_clips, filename, 'concatenate' - ) - else: - os.rename(audio_clips[0], filename) - + audio_concat = concatenate_audioclips(audio_clips) + audio_composite = CompositeAudioClip([audio_concat]) + audio_composite.write_audiofile(filename, 44100, 2, 2000, None) @staticmethod def randomvoice(): From 3be00d00d71a5c753a0a1bda57275e7fc2966a93 Mon Sep 17 00:00:00 2001 From: PatatjeMC Date: Wed, 8 Jun 2022 20:48:50 +0200 Subject: [PATCH 5/6] Changed to tts chunk combiner to sox This helps performance and improves audio quality as movie.py adds a lot of audio glitches also there where a few small mistakes in the old version that I fixed and seem to work. --- requirements.txt | 2 +- utils/subreddit.py | 3 ++- video_creation/TTSwrapper.py | 17 +++++++++++------ 3 files changed, 14 insertions(+), 8 deletions(-) diff --git a/requirements.txt b/requirements.txt index 0578bfb..89a66c6 100644 --- a/requirements.txt +++ b/requirements.txt @@ -6,4 +6,4 @@ playwright~=1.22.0 python-dotenv==0.20.0 typed-ast~=1.5.4 requests~=2.27.1 -yt-dlp +typing~=3.10.0.0 diff --git a/utils/subreddit.py b/utils/subreddit.py index 7fd7117..68840d2 100644 --- a/utils/subreddit.py +++ b/utils/subreddit.py @@ -1,4 +1,5 @@ -from typings import List +from typing import List +import json def get_hottest_undone(submissions: List): """ recursively checks if the top submission in the list was already done. diff --git a/video_creation/TTSwrapper.py b/video_creation/TTSwrapper.py index d8fa2ec..2ebdb28 100644 --- a/video_creation/TTSwrapper.py +++ b/video_creation/TTSwrapper.py @@ -3,8 +3,8 @@ import os import random import re +import sox import requests -from moviepy.editor import AudioFileClip, concatenate_audioclips, CompositeAudioClip from requests.adapters import HTTPAdapter, Retry # from profanity_filter import ProfanityFilter @@ -88,6 +88,7 @@ class TTTTSWrapper: # TikTok Text-to-Speech Wrapper ] audio_clips = [] + cbn = sox.Combiner() chunkId = 0 for chunk in chunks: @@ -108,16 +109,20 @@ class TTTTSWrapper: # TikTok Text-to-Speech Wrapper vstr = [r.json()["data"]["v_str"]][0] b64d = base64.b64decode(vstr) - with open(f"{filename}-{chunkId}", "wb") as out: + with open(filename.replace(".mp3", f"-{chunkId}.mp3"), "wb") as out: out.write(b64d) - audio_clips.append(AudioFileClip(f"{filename}-{chunkId}")) + audio_clips.append(filename.replace(".mp3", f"-{chunkId}.mp3")) chunkId = chunkId + 1 - audio_concat = concatenate_audioclips(audio_clips) - audio_composite = CompositeAudioClip([audio_concat]) - audio_composite.write_audiofile(filename, 44100, 2, 2000, None) + if(len(audio_clips) > 1): + cbn.convert(samplerate=44100, n_channels=2) + cbn.build( + audio_clips, filename, 'concatenate' + ) + else: + os.rename(audio_clips[0], filename) @staticmethod def randomvoice(): From 1a8a668069527ce2cdd650bc7b04591c696074f7 Mon Sep 17 00:00:00 2001 From: PatatjeMC Date: Wed, 8 Jun 2022 20:53:08 +0200 Subject: [PATCH 6/6] Fixed requirements.txt pytube was removed of it idk when but it's still being used so here it is back even tho it's half broken and it should probably be replaced soon. also typing did not work with the version for some reason --- requirements.txt | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/requirements.txt b/requirements.txt index 89a66c6..41bf400 100644 --- a/requirements.txt +++ b/requirements.txt @@ -6,4 +6,5 @@ playwright~=1.22.0 python-dotenv==0.20.0 typed-ast~=1.5.4 requests~=2.27.1 -typing~=3.10.0.0 +pytube~=12.1.0 +typing