From fe082a90804f0137393859794bf3c4c844a7ed8b Mon Sep 17 00:00:00 2001 From: Hong Phuc Date: Tue, 19 May 2026 13:19:08 +0700 Subject: [PATCH] feat: improve Threads settings and add Supertonic TTS Add Supertonic as the default TTS provider, introduce browser backend plumbing, make the Settings UI template-driven, and tighten Threads discovery/auth behavior with regression coverage. --- .github/ISSUE_TEMPLATE/config.yml | 2 +- .gitignore | 2 + Dockerfile | 5 +- GUI.py | 15 ++- GUI/settings.html | 187 ++++++++++++++++++++++++++--- TTS/engine_wrapper.py | 11 +- TTS/supertonic_tts.py | 63 ++++++++++ docker-compose.yml | 6 + platforms/threads/auth.py | 71 +++++++++-- platforms/threads/scraper.py | 190 +++++++++++++++--------------- platforms/threads/screenshot.py | 10 +- requirements.txt | 2 + tests/test_browser_backend.py | 54 +++++++++ tests/test_gui_routes.py | 17 ++- tests/test_gui_utils.py | 30 +++++ tests/test_settings_defaults.py | 43 +++++++ tests/test_settings_template.py | 25 ++++ tests/test_supertonic_tts.py | 55 +++++++++ tests/test_threads_auth.py | 59 ++++++++++ tests/test_tts_engine_wrapper.py | 50 ++++++++ utils/.config.template.toml | 10 +- utils/browser_backend.py | 47 ++++++++ utils/docker_bootstrap.py | 1 + utils/gui_utils.py | 15 +++ utils/settings.py | 24 ++++ video_creation/voices.py | 2 + 26 files changed, 847 insertions(+), 149 deletions(-) create mode 100644 TTS/supertonic_tts.py create mode 100644 tests/test_browser_backend.py create mode 100644 tests/test_settings_defaults.py create mode 100644 tests/test_settings_template.py create mode 100644 tests/test_supertonic_tts.py create mode 100644 tests/test_threads_auth.py create mode 100644 tests/test_tts_engine_wrapper.py create mode 100644 utils/browser_backend.py diff --git a/.github/ISSUE_TEMPLATE/config.yml b/.github/ISSUE_TEMPLATE/config.yml index 3390cfc..a3e6618 100644 --- a/.github/ISSUE_TEMPLATE/config.yml +++ b/.github/ISSUE_TEMPLATE/config.yml @@ -2,4 +2,4 @@ blank_issues_enabled: false contact_links: - name: Ask a question about: Join our discord server to ask questions and discuss with maintainers and contributors. - url: https://discord.gg/swqtb7AsNQ \ No newline at end of file + url: https://discord.gg/swqtb7AsNQ diff --git a/.gitignore b/.gitignore index 4f610f6..dd2af95 100644 --- a/.gitignore +++ b/.gitignore @@ -270,3 +270,5 @@ agentdb.rvf.lock claude-flow.config.json ruvector.db pipeline-ui-*.png +.codex +.agents diff --git a/Dockerfile b/Dockerfile index 794e738..758642f 100644 --- a/Dockerfile +++ b/Dockerfile @@ -3,7 +3,8 @@ FROM python:3.14-slim-bookworm ENV PYTHONDONTWRITEBYTECODE=1 \ PYTHONUNBUFFERED=1 \ PIP_NO_CACHE_DIR=1 \ - PLAYWRIGHT_BROWSERS_PATH=/ms-playwright + PLAYWRIGHT_BROWSERS_PATH=/ms-playwright \ + XDG_CACHE_HOME=/app/.cache WORKDIR /app @@ -25,6 +26,8 @@ COPY . . RUN groupadd -r appuser && useradd -r -g appuser -d /app appuser \ && chown -R appuser:appuser /app /ms-playwright +ENV CLOAKBROWSER_CACHE_DIR=/app/.cache/cloakbrowser + RUN chmod +x /app/docker-entrypoint.sh USER appuser diff --git a/GUI.py b/GUI.py index 4e0cb98..2f7b6be 100644 --- a/GUI.py +++ b/GUI.py @@ -5,6 +5,7 @@ import sys import threading import time import webbrowser +from copy import deepcopy from pathlib import Path # Used "tomlkit" instead of "toml" because it doesn't change formatting on "dump" @@ -23,6 +24,7 @@ from flask import ( import utils.gui_utils as gui from utils.docker_bootstrap import ensure_runtime_state +from utils.settings import apply_template_defaults ensure_runtime_state() @@ -111,7 +113,7 @@ def _redact_secrets(data: dict) -> dict: @app.route("/settings", methods=["GET", "POST"]) def settings(): config_load = tomlkit.loads(Path("config.toml").read_text()) - config = gui.get_config(config_load) + config = gui.get_config(apply_template_defaults(deepcopy(config_load))) # Get checks for all values checks = gui.get_checks() @@ -121,7 +123,8 @@ def settings(): data = request.form.to_dict() # Change settings - config = gui.modify_settings(data, config_load, checks) + gui.modify_settings(data, config_load, checks) + config = gui.get_config(apply_template_defaults(deepcopy(config_load))) return render_template("settings.html", file="config.toml", data=_redact_secrets(config), checks=checks) @@ -240,8 +243,8 @@ def _run_pipeline(search_queries=None): pipeline_state["scraper_events"] = [] try: - # Load config - settings.config = toml.load("config.toml") + # Load config and merge template defaults for non-interactive GUI runs. + settings.config = settings.apply_template_defaults(toml.load("config.toml")) # Apply search_queries override if provided from UI if search_queries: @@ -272,10 +275,14 @@ def _run_pipeline(search_queries=None): import importlib import video_creation.final_video import video_creation.background + import video_creation.voices + import TTS.engine_wrapper import platforms.threads.screenshot import main importlib.reload(video_creation.final_video) importlib.reload(video_creation.background) + importlib.reload(TTS.engine_wrapper) + importlib.reload(video_creation.voices) importlib.reload(platforms.threads.screenshot) importlib.reload(main) diff --git a/GUI/settings.html b/GUI/settings.html index a30d366..88acfc1 100644 --- a/GUI/settings.html +++ b/GUI/settings.html @@ -82,25 +82,36 @@
/// Threads Configuration
-
-
+ + -
- - + +
+ +
+
@@ -232,13 +243,9 @@
@@ -259,15 +266,95 @@ -
/// API Credentials
+
-
+ -
+ + + +
+ + +
+ + + + + + + + + + + + +
@@ -345,6 +432,12 @@ } else { input.value = data[name]; } + } else if (validateChecks[name] && validateChecks[name].default !== undefined) { + if (input.type === 'checkbox') { + input.checked = (validateChecks[name].default === "True" || validateChecks[name].default === true); + } else { + input.value = validateChecks[name].default; + } } if (input.type === 'range') { @@ -360,6 +453,8 @@ // ---- Platform Visibility ------------------------------------------- const platformSelect = document.getElementById('platformSelect'); + const threadsDiscoveryMethodSelect = document.getElementById('threadsDiscoveryMethodSelect'); + function applyPlatformVisibility() { const current = platformSelect.value || "reddit"; document.querySelectorAll('.platform-section').forEach(section => { @@ -368,8 +463,58 @@ section.querySelectorAll('input, select, textarea').forEach(el => el.disabled = !matches); }); } + + function matchesDiscoveryMethod(section, method) { + return (section.dataset.discoveryMethods || '') + .split(',') + .map(value => value.trim().toLowerCase()) + .filter(Boolean) + .includes((method || '').trim().toLowerCase()); + } + + function applyThreadsDiscoveryVisibility() { + const isThreads = (platformSelect.value || 'reddit') === 'threads'; + const current = threadsDiscoveryMethodSelect?.value || 'api'; + document.querySelectorAll('.threads-discovery-section').forEach(section => { + const matches = isThreads && matchesDiscoveryMethod(section, current); + const preserveHidden = section.dataset.preserveHidden === 'true'; + section.classList.toggle('hidden', !matches); + section.querySelectorAll('input, select, textarea').forEach(el => { + el.disabled = !isThreads || (!matches && !preserveHidden); + }); + }); + } + platformSelect.addEventListener('change', applyPlatformVisibility); + platformSelect.addEventListener('change', applyThreadsDiscoveryVisibility); + threadsDiscoveryMethodSelect?.addEventListener('change', applyThreadsDiscoveryVisibility); applyPlatformVisibility(); + applyThreadsDiscoveryVisibility(); + + // ---- TTS Provider Visibility --------------------------------------- + const voiceChoiceSelect = document.getElementById('voiceChoiceSelect'); + + function matchesTtsProvider(section, provider) { + return (section.dataset.ttsProviders || '') + .split(',') + .map(value => value.trim().toLowerCase()) + .filter(Boolean) + .includes((provider || '').trim().toLowerCase()); + } + + function applyTtsVisibility() { + const current = voiceChoiceSelect.value || 'Supertonic'; + document.querySelectorAll('.tts-provider-section').forEach(section => { + const matches = matchesTtsProvider(section, current); + section.classList.toggle('hidden', !matches); + section.querySelectorAll('input, select, textarea').forEach(el => { + el.disabled = !matches; + }); + }); + } + + voiceChoiceSelect.addEventListener('change', applyTtsVisibility); + applyTtsVisibility(); // ---- Validation ---------------------------------------------------- function validateInput(input) { @@ -455,6 +600,10 @@ validateInput(input); } }); + + applyPlatformVisibility(); + applyThreadsDiscoveryVisibility(); + applyTtsVisibility(); }); lucide.createIcons(); diff --git a/TTS/engine_wrapper.py b/TTS/engine_wrapper.py index 71008d5..0939880 100644 --- a/TTS/engine_wrapper.py +++ b/TTS/engine_wrapper.py @@ -15,9 +15,9 @@ from utils import settings from utils.console import print_step, print_substep from utils.voice import sanitize_text -# TikTok + pyttsx3 imports — used for graceful fallback when TikTok TTS fails +# TikTok + Google Translate imports — used for graceful fallback when TikTok TTS fails +from TTS.GTTS import GTTS from TTS.TikTok import TikTokTTSException -from TTS.pyttsx import pyttsx as PyttsxModule DEFAULT_MAX_LENGTH: int = ( 50 # Video length variable, edit this on your own risk. It should work, but it's not supported @@ -159,15 +159,14 @@ class TTSEngine: ) except TikTokTTSException as err: print_substep( - f"TikTok TTS failed ({err}). Falling back to pyttsx3 for this segment.", + f"TikTok TTS failed ({err}). Falling back to Google Translate TTS for this segment.", "bold yellow", ) - settings.config["settings"]["tts"]["voice_choice"] = "pyttsx" - self.tts_module = PyttsxModule() + settings.config["settings"]["tts"]["voice_choice"] = "googletranslate" + self.tts_module = GTTS() self.tts_module.run( text, filepath=f"{self.path}/{filename}.mp3", - random_voice=settings.config["settings"]["tts"]["random_voice"], ) # try: # self.length += MP3(f"{self.path}/{filename}.mp3").info.length diff --git a/TTS/supertonic_tts.py b/TTS/supertonic_tts.py new file mode 100644 index 0000000..3264b58 --- /dev/null +++ b/TTS/supertonic_tts.py @@ -0,0 +1,63 @@ +import random +import subprocess +import tempfile +from pathlib import Path + +from supertonic import TTS + +from utils import settings + + +class SupertonicTTS: + def __init__(self): + self.max_chars = 300 + self.tts = TTS(auto_download=True) + + def run(self, text, filepath, random_voice: bool = False): + output_path = Path(filepath) + output_path.parent.mkdir(parents=True, exist_ok=True) + + tts_settings = settings.config["settings"].get("tts", {}) + voice_style = self._voice_style(tts_settings, random_voice) + + wav, _duration = self.tts.synthesize( + text, + voice_style=voice_style, + lang=tts_settings.get("supertonic_lang", "na"), + total_steps=int(tts_settings.get("supertonic_steps", 8)), + speed=float(tts_settings.get("supertonic_speed", 1.05)), + max_chunk_length=self.max_chars, + verbose=False, + ) + + with tempfile.NamedTemporaryFile(suffix=".wav", delete=False) as temp_wav: + wav_path = Path(temp_wav.name) + + try: + self.tts.save_audio(wav, str(wav_path)) + subprocess.run( + [ + "ffmpeg", + "-y", + "-hide_banner", + "-loglevel", + "error", + "-i", + str(wav_path), + str(output_path), + ], + check=True, + ) + finally: + wav_path.unlink(missing_ok=True) + + def randomvoice(self): + return random.choice(list(self.tts.voice_style_names)) + + def _voice_style(self, tts_settings: dict, random_voice: bool): + custom_voice_path = str(tts_settings.get("supertonic_custom_voice_path", "")).strip() + if custom_voice_path: + return self.tts.get_voice_style_from_path(custom_voice_path) + + voice_name = self.randomvoice() if random_voice else tts_settings.get("supertonic_voice", "M1") + return self.tts.get_voice_style(voice_name=str(voice_name)) \ No newline at end of file diff --git a/docker-compose.yml b/docker-compose.yml index 25a7372..754f659 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -11,6 +11,8 @@ services: GUI_PORT: "4000" GUI_OPEN_BROWSER: "0" GUI_BROWSER_URL: "http://localhost:4000" + XDG_CACHE_HOME: "/app/.cache" + CLOAKBROWSER_CACHE_DIR: "/app/.cache/cloakbrowser" volumes: - ./:/app shm_size: "1gb" @@ -22,6 +24,8 @@ services: command: ["python", "main.py"] environment: PYTHONUNBUFFERED: "1" + XDG_CACHE_HOME: "/app/.cache" + CLOAKBROWSER_CACHE_DIR: "/app/.cache/cloakbrowser" volumes: - ./:/app shm_size: "1gb" @@ -34,6 +38,8 @@ services: environment: PYTHONUNBUFFERED: "1" PYTHONPATH: "/app" + XDG_CACHE_HOME: "/app/.cache" + CLOAKBROWSER_CACHE_DIR: "/app/.cache/cloakbrowser" volumes: - ./:/app shm_size: "1gb" diff --git a/platforms/threads/auth.py b/platforms/threads/auth.py index d7636a8..62a1b51 100644 --- a/platforms/threads/auth.py +++ b/platforms/threads/auth.py @@ -1,27 +1,52 @@ -"""Shared Playwright authentication for Threads.net. +"""Shared Playwright authentication for Threads. Used by both the screenshotter (screenshot.py) and the web scraper (scraper.py). """ import json from pathlib import Path +from urllib.parse import urlparse -from playwright.sync_api import Browser, BrowserContext, Page, ViewportSize +from playwright.sync_api import Browser, BrowserContext, Page, TimeoutError, ViewportSize from utils import settings from utils.console import emit_scraper_event, print_substep -THREADS_LOGIN_URL = "https://www.threads.net/login" +THREADS_LOGIN_URL = "https://www.threads.com/login" THREADS_COOKIE_FILE = "./video_creation/data/cookie-threads.json" DEFAULT_USER_AGENT = ( "Mozilla/5.0 (Windows NT 10.0; Win64; x64) " "AppleWebKit/537.36 (KHTML, like Gecko) " "Chrome/120.0.0.0 Safari/537.36" ) +THREADS_HOSTS = { + "threads.com", + "www.threads.com", + "threads.net", + "www.threads.net", +} +THREADS_NON_AUTH_PATH_PREFIXES = ( + "/login", + "/challenge", + "/checkpoint", + "/consent", + "/accountsuspended", + "/suspended", +) + + +def _is_logged_in_threads_url(url: str) -> bool: + """Return True once Threads has navigated away from the login page.""" + parsed = urlparse(url) + hostname = (parsed.hostname or "").casefold() + path = (parsed.path or "/").lower() + return parsed.scheme == "https" and hostname in THREADS_HOSTS and not any( + path.startswith(prefix) for prefix in THREADS_NON_AUTH_PATH_PREFIXES + ) def login_to_threads(page: Page, _context: BrowserContext) -> None: - """Log into threads.net via Instagram credentials and persist session cookies.""" + """Log into Threads via Instagram credentials and persist session cookies.""" username = settings.config["threads"]["creds"].get("username", "").strip() password = settings.config["threads"]["creds"].get("password", "").strip() @@ -34,15 +59,35 @@ def login_to_threads(page: Page, _context: BrowserContext) -> None: print_substep("Logging into Threads (via Instagram)...") emit_scraper_event("login", {"message": "Logging into Threads (via Instagram)..."}) page.goto(THREADS_LOGIN_URL, timeout=0) - page.wait_for_load_state("networkidle") - - page.locator('input[autocomplete="username"]').fill(username) - page.locator('input[autocomplete="current-password"]').fill(password) - page.get_by_role("button", name="Log in", exact=True).first.click() - - # Wait for navigation to feed (success) or stay on login (failure) - page.wait_for_url("https://www.threads.net/", timeout=15000) - page.wait_for_load_state("networkidle") + page.wait_for_load_state("domcontentloaded") + + username_input = page.locator('input[autocomplete="username"]') + password_input = page.locator('input[autocomplete="current-password"]') + username_input.fill(username) + password_input.fill(password) + + # CloakBrowser can be slightly flaky with the humanized button click even + # when the credentials are correct. Try the visible login button first, + # then fall back to pressing Enter in the password field. + submit_attempts = [ + ("button click", lambda: page.get_by_role("button", name="Log in", exact=True).first.click()), + ("password Enter", lambda: password_input.press("Enter")), + ] + last_error = None + for index, (label, submit) in enumerate(submit_attempts): + try: + submit() + # Threads currently redirects to threads.com on successful login, + # but older saved sessions and some flows may still hit threads.net. + page.wait_for_url(_is_logged_in_threads_url, timeout=30000 if index == 0 else 15000) + break + except TimeoutError as error: + last_error = error + if index == len(submit_attempts) - 1: + raise + print_substep(f"Threads login via {label} timed out. Retrying...", style="yellow") + + page.wait_for_load_state("domcontentloaded") cookies = _context.cookies() Path(THREADS_COOKIE_FILE).parent.mkdir(parents=True, exist_ok=True) diff --git a/platforms/threads/scraper.py b/platforms/threads/scraper.py index d7728c0..3cfc715 100644 --- a/platforms/threads/scraper.py +++ b/platforms/threads/scraper.py @@ -8,10 +8,11 @@ Returns the standard content_object dict consumed by the rest of the pipeline. import re from typing import Optional -from playwright.sync_api import BrowserContext, Locator, sync_playwright +from playwright.sync_api import BrowserContext, Locator from platforms.threads.auth import ensure_authenticated_context from utils import settings +from utils.browser_backend import launch_browser from utils.console import emit_scraper_event, print_step, print_substep from utils.voice import sanitize_text from utils.videos import check_done_by_id @@ -526,108 +527,103 @@ def get_trending_threads_content(POST_ID: Optional[str] = None) -> dict: min_replies = int(settings.config["threads"]["thread"]["min_replies"]) min_engagement = int(settings.config["threads"]["thread"].get("min_engagement", 0)) - with sync_playwright() as p: - browser = p.chromium.launch(headless=True) - try: - context = ensure_authenticated_context(browser) - - if POST_ID: - post_url = f"https://www.threads.net/t/{POST_ID}" - post = {"url": post_url, "post_id": POST_ID, "text": "", "body": ""} - replies = _scrape_post_replies(context, post_url) - content = _build_content_object(post, replies) - if content["comments"] or content.get("thread_post"): - return content - raise RuntimeError( - f"No replies found for post {POST_ID}. " - f"Minimum required: {min_replies}." - ) + with launch_browser(headless=True) as browser: + context = ensure_authenticated_context(browser) - # Scrape from multiple sources: main feed + trending search queries - posts = _scrape_feed_posts(context) - # Also search for popular topics to find high-engagement content - trending_queries = settings.config["threads"]["thread"].get( - "search_queries", "news,politics,trending" + if POST_ID: + post_url = f"https://www.threads.net/t/{POST_ID}" + post = {"url": post_url, "post_id": POST_ID, "text": "", "body": ""} + replies = _scrape_post_replies(context, post_url) + content = _build_content_object(post, replies) + if content["comments"] or content.get("thread_post"): + return content + raise RuntimeError( + f"No replies found for post {POST_ID}. " + f"Minimum required: {min_replies}." ) - for query in trending_queries.split(","): - query = query.strip() - if query: - try: - search_posts = _scrape_search_page(context, query) - # Merge avoiding duplicates - existing_ids = {p["post_id"] for p in posts} - for sp in search_posts: - if sp["post_id"] not in existing_ids: - posts.append(sp) - except Exception as e: - print_substep(f"Search query failed: {e}", "yellow") - - if not posts: - raise RuntimeError("No posts found in feed. Try again later.") - - candidates = _filter_candidates(posts) - if not candidates: - raise RuntimeError( - f"No eligible posts in feed after filtering. " - f"Try lowering min_engagement (currently {min_engagement:,}) " - f"or min_replies (currently {min_replies})." - ) - for i, candidate in enumerate(candidates): - eng = candidate.get("_total_engagement", 0) - print_substep( - f"Trying #{i + 1}: ♥{candidate['likes']:,} " - f"💬{candidate['replies_shown']} " - f"'{candidate['body'][:60]}...'", - style="dim", - ) - emit_scraper_event("visiting_post", { - "post_id": candidate["post_id"], - "url": candidate["url"], - "engagement": eng, - "likes": candidate.get("likes", 0), - "body": candidate.get("body", "")[:60], - "attempt": i + 1, - }) + # Scrape from multiple sources: main feed + trending search queries + posts = _scrape_feed_posts(context) + # Also search for popular topics to find high-engagement content + trending_queries = settings.config["threads"]["thread"].get( + "search_queries", "news,politics,trending" + ) + for query in trending_queries.split(","): + query = query.strip() + if query: try: - replies = _scrape_post_replies(context, candidate["url"]) - emit_scraper_event("replies_found", { - "post_id": candidate["post_id"], - "count": len(replies), - "min_required": min_replies, - }) - if len(replies) >= min_replies: - if not candidate.get("body") or len(candidate.get("body", "")) < 50: - full_text = _scrape_main_post_text(context, candidate["url"]) - if full_text: - candidate["body"] = full_text - content = _build_content_object(candidate, replies) - title_preview = content["thread_title"][:60] - print_substep( - f"Selected: '{title_preview}...' " - f"♥{candidate['likes']:,} 💬{len(content['comments'])} replies", - style="bold green", - ) - emit_scraper_event("post_selected", { - "title": content["thread_title"][:80], - "post_id": candidate["post_id"], - "likes": candidate["likes"], - "replies_count": len(content["comments"]), - "url": candidate["url"], - }) - return content - print_substep( - f" Only {len(replies)} replies (need {min_replies}). Trying next...", - style="yellow", - ) + search_posts = _scrape_search_page(context, query) + # Merge avoiding duplicates + existing_ids = {p["post_id"] for p in posts} + for sp in search_posts: + if sp["post_id"] not in existing_ids: + posts.append(sp) except Exception as e: - print_substep(f" Failed: {e}. Trying next...", style="yellow") - continue + print_substep(f"Search query failed: {e}", "yellow") + if not posts: + raise RuntimeError("No posts found in feed. Try again later.") + + candidates = _filter_candidates(posts) + if not candidates: raise RuntimeError( - f"No eligible posts with {min_replies}+ replies found " - f"after trying {len(candidates)} candidates." + f"No eligible posts in feed after filtering. " + f"Try lowering min_engagement (currently {min_engagement:,}) " + f"or min_replies (currently {min_replies})." ) - finally: - browser.close() + for i, candidate in enumerate(candidates): + eng = candidate.get("_total_engagement", 0) + print_substep( + f"Trying #{i + 1}: ♥{candidate['likes']:,} " + f"💬{candidate['replies_shown']} " + f"'{candidate['body'][:60]}...'", + style="dim", + ) + emit_scraper_event("visiting_post", { + "post_id": candidate["post_id"], + "url": candidate["url"], + "engagement": eng, + "likes": candidate.get("likes", 0), + "body": candidate.get("body", "")[:60], + "attempt": i + 1, + }) + try: + replies = _scrape_post_replies(context, candidate["url"]) + emit_scraper_event("replies_found", { + "post_id": candidate["post_id"], + "count": len(replies), + "min_required": min_replies, + }) + if len(replies) >= min_replies: + if not candidate.get("body") or len(candidate.get("body", "")) < 50: + full_text = _scrape_main_post_text(context, candidate["url"]) + if full_text: + candidate["body"] = full_text + content = _build_content_object(candidate, replies) + title_preview = content["thread_title"][:60] + print_substep( + f"Selected: '{title_preview}...' " + f"♥{candidate['likes']:,} 💬{len(content['comments'])} replies", + style="bold green", + ) + emit_scraper_event("post_selected", { + "title": content["thread_title"][:80], + "post_id": candidate["post_id"], + "likes": candidate["likes"], + "replies_count": len(content["comments"]), + "url": candidate["url"], + }) + return content + print_substep( + f" Only {len(replies)} replies (need {min_replies}). Trying next...", + style="yellow", + ) + except Exception as e: + print_substep(f" Failed: {e}. Trying next...", style="yellow") + continue + + raise RuntimeError( + f"No eligible posts with {min_replies}+ replies found " + f"after trying {len(candidates)} candidates." + ) diff --git a/platforms/threads/screenshot.py b/platforms/threads/screenshot.py index 2115b0f..b24128e 100644 --- a/platforms/threads/screenshot.py +++ b/platforms/threads/screenshot.py @@ -1,13 +1,14 @@ -"""Captures screenshots of Threads posts via Playwright.""" +"""Captures screenshots of Threads posts via the configured browser backend.""" import re from pathlib import Path from typing import Final -from playwright.sync_api import ViewportSize, sync_playwright +from playwright.sync_api import ViewportSize from platforms.threads.auth import ensure_authenticated_context from utils import settings +from utils.browser_backend import launch_browser from utils.console import print_step, print_substep @@ -40,9 +41,8 @@ def get_screenshots_of_threads_posts(content_object: dict, screenshot_num: int) # Device scale factor (higher resolution screenshots) dsf = (W // 600) + 1 - with sync_playwright() as p: + with launch_browser(headless=True) as browser: print_substep("Launching headless browser...") - browser = p.chromium.launch(headless=True) context = ensure_authenticated_context( browser, color_scheme="dark" if theme == "dark" else "light", @@ -153,6 +153,4 @@ def get_screenshots_of_threads_posts(content_object: dict, screenshot_num: int) print_substep(f"Reply screenshots captured ({num_replies} total).", style="bold green") - browser.close() - print_substep("Threads screenshots downloaded successfully.", style="bold green") diff --git a/requirements.txt b/requirements.txt index af1f97c..7feaac5 100644 --- a/requirements.txt +++ b/requirements.txt @@ -3,6 +3,7 @@ botocore==1.43.3 gTTS==2.5.4 moviepy==2.2.1 playwright==1.59.0 +cloakbrowser==0.3.28 pyotp==2.9.0 praw==7.8.1 requests==2.33.1 @@ -16,6 +17,7 @@ clean-text==0.7.1 unidecode==1.4.0 torch==2.11.0 transformers==4.57.6 +supertonic==1.3.1 spacy==3.8.13 av>=14.0 elevenlabs==2.44.0 diff --git a/tests/test_browser_backend.py b/tests/test_browser_backend.py new file mode 100644 index 0000000..cadfbe3 --- /dev/null +++ b/tests/test_browser_backend.py @@ -0,0 +1,54 @@ +import sys +from types import SimpleNamespace + +from utils import settings + + +class FakeBrowser: + def __init__(self): + self.closed = False + + def close(self): + self.closed = True + + +def test_launch_browser_defaults_to_playwright(monkeypatch): + from utils import browser_backend + + fake_browser = FakeBrowser() + + class FakePlaywrightContext: + def __enter__(self): + return SimpleNamespace( + chromium=SimpleNamespace(launch=lambda headless: fake_browser) + ) + + def __exit__(self, exc_type, exc, traceback): + return False + + settings.config = {"settings": {}} + monkeypatch.setattr(browser_backend, "sync_playwright", lambda: FakePlaywrightContext()) + + with browser_backend.launch_browser() as browser: + assert browser is fake_browser + + assert fake_browser.closed is True + + +def test_launch_browser_uses_cloakbrowser_when_configured(monkeypatch): + from utils import browser_backend + + fake_browser = FakeBrowser() + launch_calls = [] + fake_module = SimpleNamespace( + launch=lambda **kwargs: launch_calls.append(kwargs) or fake_browser + ) + + settings.config = {"settings": {"browser": {"backend": "cloakbrowser"}}} + monkeypatch.setitem(sys.modules, "cloakbrowser", fake_module) + + with browser_backend.launch_browser(headless=False) as browser: + assert browser is fake_browser + + assert launch_calls == [{"headless": False, "humanize": True}] + assert fake_browser.closed is True \ No newline at end of file diff --git a/tests/test_gui_routes.py b/tests/test_gui_routes.py index dceadcb..fc07985 100644 --- a/tests/test_gui_routes.py +++ b/tests/test_gui_routes.py @@ -1,4 +1,4 @@ -from unittest.mock import patch +from unittest.mock import MagicMock, patch from GUI import app @@ -28,3 +28,18 @@ def test_background_add_passes_empty_defaults_for_missing_optional_fields(): "", "", ) + + +def test_settings_get_uses_template_defaults_for_partial_config(): + app.testing = True + fake_path = MagicMock() + fake_path.read_text.return_value = "" + + with patch("GUI.Path", MagicMock(return_value=fake_path)): + response = app.test_client().get("/settings") + + body = response.get_data(as_text=True) + + assert response.status_code == 200 + assert '"settings.tts.voice_choice": "Supertonic"' in body + assert 'name="settings.tts.supertonic_voice"' in body diff --git a/tests/test_gui_utils.py b/tests/test_gui_utils.py index 19b4ed8..3bafbb5 100644 --- a/tests/test_gui_utils.py +++ b/tests/test_gui_utils.py @@ -99,3 +99,33 @@ def test_modify_settings_preserves_masked_secrets(mock_flash): assert config_load["reddit"]["creds"]["client_secret"] == "real-secret" assert config_load["reddit"]["creds"]["password"] == "changed-password" assert result["reddit.creds.client_secret"] == "real-secret" + + +@patch("utils.gui_utils.flash") +def test_modify_settings_does_not_materialize_missing_default_values(mock_flash, tmp_path): + config_load = {"settings": {"channel_name": "My Channel"}} + checks = { + "settings.channel_name": {"optional": True, "type": "str"}, + "settings.tts.voice_choice": {"optional": False, "type": "str", "default": "Supertonic"}, + "settings.tts.supertonic_voice": {"optional": True, "type": "str", "default": "M1"}, + "threads.creds.username": {"optional": True, "type": "str"}, + } + config_path = tmp_path / "config.toml" + + with patch("utils.gui_utils.Path", MagicMock(return_value=config_path)): + result = gui_utils.modify_settings( + { + "settings.channel_name": "Updated Channel", + "settings.tts.voice_choice": "Supertonic", + "settings.tts.supertonic_voice": "M1", + "threads.creds.username": "", + }, + config_load, + checks, + ) + + assert config_load["settings"]["channel_name"] == "Updated Channel" + assert "tts" not in config_load["settings"] + assert "threads" not in config_load + assert "settings.tts.voice_choice" not in result + assert "threads.creds.username" not in result diff --git a/tests/test_settings_defaults.py b/tests/test_settings_defaults.py new file mode 100644 index 0000000..96580e7 --- /dev/null +++ b/tests/test_settings_defaults.py @@ -0,0 +1,43 @@ +from utils.settings import apply_template_defaults + + +def test_apply_template_defaults_fills_missing_values_without_overwriting(tmp_path): + template = tmp_path / "template.toml" + template.write_text( + """ +[settings] +platform = { optional = false, default = "reddit" } + +[settings.tts] +no_emojis = { optional = false, default = false, type = "bool" } + +[threads.creds] +access_token = { optional = false } +username = { optional = true } + +[threads.thread] +min_replies = { optional = false, default = 5, type = "int" } +search_queries = { optional = true, default = "news,politics,trending" } +""" + ) + config = { + "settings": {"platform": "threads"}, + "threads": {"creds": {"username": "configured-user"}}, + } + + result = apply_template_defaults(config, str(template)) + + assert result["settings"]["platform"] == "threads" + assert result["settings"]["tts"]["no_emojis"] is False + assert result["threads"]["thread"]["min_replies"] == 5 + assert result["threads"]["thread"]["search_queries"] == "news,politics,trending" + assert result["threads"]["creds"]["username"] == "configured-user" + assert "access_token" not in result["threads"]["creds"] + + +def test_real_template_defaults_to_supertonic_tts(): + result = apply_template_defaults({"settings": {}}) + + assert result["settings"]["tts"]["voice_choice"] == "Supertonic" + assert result["settings"]["tts"]["supertonic_voice"] == "M1" + assert result["settings"]["tts"]["supertonic_lang"] == "na" \ No newline at end of file diff --git a/tests/test_settings_template.py b/tests/test_settings_template.py new file mode 100644 index 0000000..8522bbd --- /dev/null +++ b/tests/test_settings_template.py @@ -0,0 +1,25 @@ +from pathlib import Path + + +def test_audio_settings_template_supports_supertonic_and_provider_visibility(): + template = Path("GUI/settings.html").read_text() + + assert 'checks["settings.tts.voice_choice"]["options"]' in template + assert 'data-tts-providers="Supertonic"' in template + assert 'data-tts-providers="elevenlabs,OpenAI,tiktok"' in template + assert 'const voiceChoiceSelect = document.getElementById(\'voiceChoiceSelect\');' in template + assert 'applyTtsVisibility();' in template + + +def test_threads_settings_template_supports_discovery_specific_credentials(): + template = Path("GUI/settings.html").read_text() + + assert 'id="threadsDiscoveryMethodSelect"' in template + assert 'data-discovery-methods="api"' in template + assert 'data-discovery-methods="scrape"' in template + assert 'data-preserve-hidden="true"' in template + assert 'name="threads.creds.user_id"' in template + assert 'Screenshots still reuse your saved Threads username/password.' in template + assert "el.disabled = !isThreads || (!matches && !preserveHidden);" in template + assert 'const threadsDiscoveryMethodSelect = document.getElementById(\'threadsDiscoveryMethodSelect\');' in template + assert 'applyThreadsDiscoveryVisibility();' in template \ No newline at end of file diff --git a/tests/test_supertonic_tts.py b/tests/test_supertonic_tts.py new file mode 100644 index 0000000..d26f6b1 --- /dev/null +++ b/tests/test_supertonic_tts.py @@ -0,0 +1,55 @@ +from pathlib import Path + +from utils import settings + + +class FakeSupertonicEngine: + voice_style_names = ["M1", "F1"] + + def __init__(self, auto_download=True): + self.auto_download = auto_download + + def get_voice_style(self, voice_name): + return {"voice_name": voice_name} + + def get_voice_style_from_path(self, path): + return {"path": path} + + def synthesize(self, text, **kwargs): + return b"wav-data", 1.0 + + def save_audio(self, wav, path): + Path(path).write_bytes(wav) + + +def test_supertonic_tts_converts_wav_to_mp3(monkeypatch, tmp_path): + import TTS.supertonic_tts as supertonic_module + + run_calls = [] + + def fake_run(command, check): + run_calls.append((command, check)) + Path(command[-1]).write_bytes(b"mp3-data") + + settings.config = { + "settings": { + "tts": { + "supertonic_voice": "M1", + "supertonic_lang": "na", + "supertonic_steps": 8, + "supertonic_speed": 1.05, + "supertonic_custom_voice_path": "", + } + } + } + monkeypatch.setattr(supertonic_module, "TTS", FakeSupertonicEngine) + monkeypatch.setattr(supertonic_module.subprocess, "run", fake_run) + + output_path = tmp_path / "voice.mp3" + supertonic_module.SupertonicTTS().run("hello", str(output_path)) + + assert output_path.read_bytes() == b"mp3-data" + command, check = run_calls[0] + assert command[:5] == ["ffmpeg", "-y", "-hide_banner", "-loglevel", "error"] + assert command[-1] == str(output_path) + assert check is True \ No newline at end of file diff --git a/tests/test_threads_auth.py b/tests/test_threads_auth.py new file mode 100644 index 0000000..f47a4e5 --- /dev/null +++ b/tests/test_threads_auth.py @@ -0,0 +1,59 @@ +from unittest.mock import MagicMock, patch + +from playwright.sync_api import TimeoutError + +from platforms.threads import auth +from platforms.threads.auth import _is_logged_in_threads_url + + +def test_logged_in_threads_url_accepts_current_threads_com_home(): + assert _is_logged_in_threads_url("https://www.threads.com/") is True + assert _is_logged_in_threads_url("https://www.threads.com/?hl=en") is True + + +def test_logged_in_threads_url_accepts_legacy_threads_net_home(): + assert _is_logged_in_threads_url("https://www.threads.net/") is True + + +def test_logged_in_threads_url_rejects_login_page_and_other_hosts(): + assert _is_logged_in_threads_url("https://www.threads.com/login") is False + assert _is_logged_in_threads_url("https://www.threads.net/login") is False + assert _is_logged_in_threads_url("https://example.com/") is False + + +def test_logged_in_threads_url_rejects_checkpoint_and_challenge_paths(): + assert _is_logged_in_threads_url("https://www.threads.com/challenge/") is False + assert _is_logged_in_threads_url("https://www.threads.net/checkpoint/") is False + assert _is_logged_in_threads_url("https://www.threads.com/consent/") is False + + +def test_logged_in_threads_url_requires_https(): + assert _is_logged_in_threads_url("http://www.threads.com/") is False + + +def test_login_to_threads_retries_with_enter_when_button_click_times_out(tmp_path): + username_input = MagicMock() + password_input = MagicMock() + login_button = MagicMock() + login_button.click.side_effect = TimeoutError("button click timed out") + role_result = MagicMock() + role_result.first = login_button + + page = MagicMock() + page.locator.side_effect = lambda selector: username_input if "username" in selector else password_input + page.get_by_role.return_value = role_result + + context = MagicMock() + context.cookies.return_value = [{"name": "sessionid", "value": "ok"}] + + cookie_path = tmp_path / "cookie-threads.json" + + with patch.object(auth.settings, "config", {"threads": {"creds": {"username": "demo", "password": "secret"}}}), \ + patch.object(auth, "THREADS_COOKIE_FILE", str(cookie_path)): + auth.login_to_threads(page, context) + + username_input.fill.assert_called_once_with("demo") + password_input.fill.assert_called_once_with("secret") + password_input.press.assert_called_once_with("Enter") + assert page.wait_for_url.call_count == 1 + assert cookie_path.exists() \ No newline at end of file diff --git a/tests/test_tts_engine_wrapper.py b/tests/test_tts_engine_wrapper.py new file mode 100644 index 0000000..6c510db --- /dev/null +++ b/tests/test_tts_engine_wrapper.py @@ -0,0 +1,50 @@ +from pathlib import Path + +from TTS.TikTok import TikTokTTSException +from TTS.engine_wrapper import TTSEngine +from utils import settings + + +class FailingTikTok: + max_chars = 5000 + + def run(self, text, filepath, random_voice=False): + raise TikTokTTSException(1, "failed") + + +class FakeGTTS: + max_chars = 5000 + + def run(self, text, filepath): + Path(filepath).write_text(text) + + +class FakeAudioFileClip: + duration = 1.0 + + def __init__(self, path): + self.path = path + + def close(self): + pass + + +def test_tiktok_fallback_uses_googletranslate_not_pyttsx(monkeypatch, tmp_path): + import TTS.engine_wrapper as engine_wrapper + + settings.config = { + "settings": { + "tts": {"voice_choice": "tiktok", "random_voice": False}, + } + } + reddit_object = {"thread_id": "abc", "thread_title": "title", "comments": []} + monkeypatch.setattr(engine_wrapper, "GTTS", FakeGTTS) + monkeypatch.setattr(engine_wrapper, "AudioFileClip", FakeAudioFileClip) + + engine = TTSEngine(FailingTikTok, reddit_object, path=f"{tmp_path}/") + Path(engine.path).mkdir(parents=True) + + engine.call_tts("title", "hello") + + assert settings.config["settings"]["tts"]["voice_choice"] == "googletranslate" + assert Path(engine.path, "title.mp3").read_text() == "hello" \ No newline at end of file diff --git a/utils/.config.template.toml b/utils/.config.template.toml index 3a8b96e..849f895 100644 --- a/utils/.config.template.toml +++ b/utils/.config.template.toml @@ -64,6 +64,9 @@ resolution_h = { optional = false, default = 1920, example = 2560, explantation zoom = { optional = true, default = 1, example = 1.1, explanation = "Sets the browser zoom level. Useful if you want the text larger.", type = "float", nmin = 0.1, nmax = 2, oob_error = "The text is really difficult to read at a zoom level higher than 2" } channel_name = { optional = true, default = "Reddit Tales", example = "Reddit Stories", explanation = "Sets the channel name for the video" } +[settings.browser] +backend = { optional = true, default = "playwright", options = ["playwright", "cloakbrowser"], type = "str", explanation = "Browser backend for Threads scraping/screenshots. Defaults to Playwright; CloakBrowser is opt-in." } + [settings.background] background_video = { optional = true, default = "minecraft", example = "rocket-league", options = ["minecraft", "gta", "rocket-league", "motor-gta", "csgo-surf", "cluster-truck", "minecraft-2","multiversus","fall-guys","steep", "black", ""], explanation = "Sets the background for the video based on game name" } background_audio = { optional = true, default = "lofi", example = "chill-summer", options = ["lofi","lofi-2","chill-summer",""], explanation = "Sets the background audio for the video" } @@ -75,7 +78,7 @@ background_thumbnail_font_size = { optional = true, type = "int", default = 96, background_thumbnail_font_color = { optional = true, default = "255,255,255", example = "255,255,255", explanation = "Font color in RGB format for the thumbnail text" } [settings.tts] -voice_choice = { optional = false, default = "tiktok", options = ["elevenlabs", "streamlabspolly", "tiktok", "googletranslate", "awspolly", "pyttsx", "OpenAI"], example = "tiktok", explanation = "The voice platform used for TTS generation. " } +voice_choice = { optional = false, default = "Supertonic", options = ["elevenlabs", "streamlabspolly", "tiktok", "googletranslate", "awspolly", "pyttsx", "OpenAI", "Supertonic"], example = "Supertonic", explanation = "The voice platform used for TTS generation. " } random_voice = { optional = false, type = "bool", default = true, example = true, options = [true, false,], explanation = "Randomizes the voice used for each comment" } elevenlabs_voice_name = { optional = false, default = "Bella", example = "Bella", explanation = "The voice used for elevenlabs", options = ["Adam", "Antoni", "Arnold", "Bella", "Domi", "Elli", "Josh", "Rachel", "Sam", ] } elevenlabs_api_key = { optional = true, example = "21f13f91f54d741e2ae27d2ab1b99d59", explanation = "Elevenlabs API key" } @@ -91,3 +94,8 @@ openai_api_url = { optional = true, default = "https://api.openai.com/v1/", exam openai_api_key = { optional = true, example = "sk-abc123def456...", explanation = "Your OpenAI API key for TTS generation" } openai_voice_name = { optional = false, default = "alloy", example = "alloy", explanation = "The voice used for OpenAI TTS generation", options = ["alloy", "ash", "coral", "echo", "fable", "onyx", "nova", "sage", "shimmer", "af_heart"] } openai_model = { optional = false, default = "tts-1", example = "tts-1", explanation = "The model variant used for OpenAI TTS generation", options = ["tts-1", "tts-1-hd", "gpt-4o-mini-tts"] } +supertonic_voice = { optional = true, default = "M1", example = "M1", explanation = "The built-in Supertonic voice style.", options = ["M1", "M2", "M3", "M4", "M5", "F1", "F2", "F3", "F4", "F5"] } +supertonic_lang = { optional = true, default = "na", example = "na", explanation = "Language code passed to Supertonic synthesis." } +supertonic_steps = { optional = true, default = 8, example = 8, type = "int", nmin = 5, nmax = 12, explanation = "Supertonic quality steps. Higher is slower." } +supertonic_speed = { optional = true, default = 1.05, example = 1.05, type = "float", nmin = 0.7, nmax = 2.0, explanation = "Supertonic speech speed." } +supertonic_custom_voice_path = { optional = true, default = "", example = "", explanation = "Optional path to a Supertonic custom voice style JSON." } diff --git a/utils/browser_backend.py b/utils/browser_backend.py new file mode 100644 index 0000000..756fe59 --- /dev/null +++ b/utils/browser_backend.py @@ -0,0 +1,47 @@ +"""Optional browser backend adapter for Playwright-compatible browsers.""" + +from contextlib import contextmanager +from typing import Iterator + +from playwright.sync_api import Browser, sync_playwright + +from utils import settings + + +def _configured_backend() -> str: + config = settings.config if isinstance(settings.config, dict) else {} + browser_config = config.get("settings", {}).get("browser", {}) + return str(browser_config.get("backend", "playwright")).casefold() + + +@contextmanager +def launch_browser(headless: bool = True) -> Iterator[Browser]: + """Launch the configured browser backend. + + Defaults to stock Playwright. CloakBrowser is opt-in via + settings.browser.backend = "cloakbrowser" and returns a Playwright-compatible + Browser object, preserving browser.new_context(...) call sites. + """ + backend = _configured_backend() + + if backend == "cloakbrowser": + import cloakbrowser + + browser = cloakbrowser.launch(headless=headless, humanize=True) + try: + yield browser + finally: + browser.close() + return + + if backend != "playwright": + raise ValueError( + f"Unsupported browser backend '{backend}'. Use 'playwright' or 'cloakbrowser'." + ) + + with sync_playwright() as playwright: + browser = playwright.chromium.launch(headless=headless) + try: + yield browser + finally: + browser.close() \ No newline at end of file diff --git a/utils/docker_bootstrap.py b/utils/docker_bootstrap.py index 9c3f326..a75772b 100644 --- a/utils/docker_bootstrap.py +++ b/utils/docker_bootstrap.py @@ -52,6 +52,7 @@ def ensure_runtime_state() -> None: "assets/temp", "assets/backgrounds/audio", "assets/backgrounds/video", + ".cache", "results", "video_creation/data", ): diff --git a/utils/gui_utils.py b/utils/gui_utils.py index a4e85e1..85a4f54 100644 --- a/utils/gui_utils.py +++ b/utils/gui_utils.py @@ -124,11 +124,21 @@ def modify_settings(data: dict, config_load, checks: dict): cursor = cursor[part] cursor[parts[-1]] = value + def has_path(obj: dict, dotted_path: str) -> bool: + cursor = obj + for part in dotted_path.split("."): + if not isinstance(cursor, dict) or part not in cursor: + return False + cursor = cursor[part] + return True + # Filter data to only include keys present in checks data = {key: value for key, value in data.items() if key in checks.keys()} # Validate and apply values for name, raw_value in data.items(): + existed_before = has_path(config_load, name) + if is_sensitive_setting(name) and raw_value == MASKED_SECRET_VALUE: continue @@ -138,6 +148,11 @@ def modify_settings(data: dict, config_load, checks: dict): if value == "Error": flash("Some values were incorrect and didn't save!", "error") else: + if not existed_before: + if checks[name].get("default") == value: + continue + if checks[name].get("optional", False) and value == "": + continue # Value is valid set_by_path(config_load, name, value) diff --git a/utils/settings.py b/utils/settings.py index 45ef34f..c4fa1ba 100755 --- a/utils/settings.py +++ b/utils/settings.py @@ -13,6 +13,30 @@ config = dict # autocomplete _TYPE_COERCION = {"int": int, "float": float, "bool": bool, "str": str} +def apply_template_defaults(config_data: Dict, template_file="utils/.config.template.toml") -> Dict: + """Fill missing config values from template defaults without prompting. + + The CLI uses check_toml(), which can prompt interactively and then writes a + fully-populated config.toml. The GUI cannot prompt inside its background + thread, so it needs a non-interactive way to make partial configs usable. + Values already present in config_data are never overwritten. + """ + template = toml.load(template_file) + + def merge_defaults(target: dict, schema: dict) -> None: + for key, value in schema.items(): + if isinstance(value, dict) and "optional" in value: + if key not in target and "default" in value: + target[key] = value["default"] + elif isinstance(value, dict): + section = target.setdefault(key, {}) + if isinstance(section, dict): + merge_defaults(section, value) + + merge_defaults(config_data, template) + return config_data + + def crawl(obj: dict, func=lambda x, y: print(x, y, end="\n"), path=None): if path is None: # path Default argument value is mutable path = [] diff --git a/video_creation/voices.py b/video_creation/voices.py index 3d48e9e..3803bf5 100644 --- a/video_creation/voices.py +++ b/video_creation/voices.py @@ -9,6 +9,7 @@ from TTS.GTTS import GTTS from TTS.openai_tts import OpenAITTS from TTS.pyttsx import pyttsx from TTS.streamlabs_polly import StreamlabsPolly +from TTS.supertonic_tts import SupertonicTTS from TTS.TikTok import TikTok from utils import settings from utils.console import print_step, print_table @@ -23,6 +24,7 @@ TTSProviders = { "pyttsx": pyttsx, "ElevenLabs": elevenlabs, "OpenAI": OpenAITTS, + "Supertonic": SupertonicTTS, }