Merge a75a89ef09 into 569f25098a
Hong Phuc
23 hours ago
committed by
GitHub
commit
157b06e9be
@ -1,2 +1,20 @@
|
||||
Dockerfile
|
||||
results
|
||||
.git
|
||||
.github
|
||||
.omx
|
||||
.venv
|
||||
venv
|
||||
__pycache__
|
||||
*.pyc
|
||||
*.pyo
|
||||
*.pyd
|
||||
.pytest_cache
|
||||
.mypy_cache
|
||||
.ruff_cache
|
||||
.DS_Store
|
||||
config.toml
|
||||
results
|
||||
assets/temp
|
||||
assets/backgrounds
|
||||
video_creation/data/videos.json
|
||||
video_creation/data/cookie-threads.json
|
||||
out
|
||||
|
||||
@ -1 +1 @@
|
||||
3.10
|
||||
3.14.4
|
||||
|
||||
@ -0,0 +1,446 @@
|
||||
# CLAUDE.md — VideoMakerBot Development Guide
|
||||
|
||||
## Project Overview
|
||||
|
||||
**VideoMakerBot** — Automated short-form video creator from social media content.
|
||||
|
||||
**Status:** Production (v3.4.0)
|
||||
**Language:** Python 3.14+ (host + Docker)
|
||||
**Runtime:** **Docker only** — CLI, GUI, test go through `docker compose`. Never `python` on host.
|
||||
**Platforms:** Reddit (PRAW API), Threads (Graph API + Web Scraping)
|
||||
|
||||
### Core Mission
|
||||
Transforms social media threads (post + comments/replies) into short-form videos:
|
||||
- AI-generated speech (7+ TTS providers)
|
||||
- UI screenshots (Playwright, headless Chromium in image)
|
||||
- Background video/audio overlays
|
||||
- FFmpeg composition & output (Linux ffmpeg, full filter set + `drawtext`)
|
||||
- Optional YouTube upload
|
||||
- Web UI (Tailwind CSS + DaisyUI + Lucide + vanilla ES6) on `localhost:4000`
|
||||
|
||||
---
|
||||
|
||||
## Architecture at a Glance
|
||||
|
||||
```
|
||||
main.py (CLI)
|
||||
↓ [platform factory]
|
||||
├─→ reddit/subreddit.py [PRAW API]
|
||||
└─→ platforms/threads/
|
||||
├─→ fetcher.py [Graph API — your own posts]
|
||||
├─→ scraper.py [Web scraping — trending For You feed]
|
||||
└─→ auth.py [Shared Playwright login + cookies]
|
||||
↓ [standard data dict]
|
||||
├─→ TTS/engine_wrapper.py [7+ providers, auto-fallback]
|
||||
├─→ screenshot_downloader.py (Reddit)
|
||||
│ or platforms/threads/screenshot.py (Threads)
|
||||
├─→ video_creation/background.py [local or yt-dlp]
|
||||
├─→ video_creation/youtube_uploader.py [optional auto-upload]
|
||||
└─→ video_creation/final_video.py [FFmpeg with libx264; exports get_output_path()]
|
||||
↓
|
||||
results/{category}/{video.mp4}
|
||||
```
|
||||
|
||||
---
|
||||
|
||||
## Data Contract: The "content_object" Dict
|
||||
|
||||
All fetchers return this shape:
|
||||
|
||||
```python
|
||||
{
|
||||
"thread_id": str, # Used for temp folder: assets/temp/{id}/
|
||||
"thread_category": str, # "reddit", "threads" → output folder
|
||||
"thread_title": str, # TTS + output filename (clean, no metadata)
|
||||
"thread_url": str, # Playwright navigates here for screenshot
|
||||
"is_nsfw": bool,
|
||||
"comments": [
|
||||
{
|
||||
"comment_body": str, # TTS per reply (clean body text)
|
||||
"comment_url": str, # Playwright navigates here
|
||||
"comment_id": str, # Unique identifier (URL-based for scraper)
|
||||
}
|
||||
],
|
||||
"thread_post": str | list, # Story mode (no comments)
|
||||
}
|
||||
```
|
||||
|
||||
---
|
||||
|
||||
## File Organization
|
||||
|
||||
```
|
||||
VideoMakerBot/
|
||||
├── platforms/
|
||||
│ ├── __init__.py # Factory: get_content_object(), get_screenshot_fn()
|
||||
│ └── threads/
|
||||
│ ├── auth.py # Shared Playwright login + cookie management
|
||||
│ ├── fetcher.py # Graph API → content_object (your own posts)
|
||||
│ ├── scraper.py # Web scraping → content_object (trending feed)
|
||||
│ └── screenshot.py # Playwright Threads screenshotter (div-based)
|
||||
│
|
||||
├── reddit/
|
||||
│ └── subreddit.py # PRAW API → content_object
|
||||
│
|
||||
├── video_creation/
|
||||
│ ├── final_video.py # FFmpeg composition (libx264, no drawtext on macOS)
|
||||
│ ├── background.py # Video/audio downloader (local files or yt-dlp)
|
||||
│ ├── screenshot_downloader.py # Playwright Reddit UI capturer
|
||||
│ ├── voices.py # TTS orchestrator
|
||||
│ └── youtube_uploader.py # YouTube OAuth2 upload (post-render hook)
|
||||
│
|
||||
├── TTS/
|
||||
│ ├── engine_wrapper.py # Provider abstraction + TikTok→pyttsx3 fallback
|
||||
│ ├── TikTok.py # TikTok TTS (hardened error handling)
|
||||
│ └── ... # 7+ provider implementations
|
||||
│
|
||||
├── utils/
|
||||
│ ├── settings.py # Config loading + interactive validation
|
||||
│ ├── videos.py # check_done() + check_done_by_id()
|
||||
│ ├── console.py # Rich terminal output
|
||||
│ ├── .config.template.toml # Config schema
|
||||
│ ├── background_videos.json # Background video manifest
|
||||
│ ├── background_audios.json # Background audio manifest
|
||||
│ └── ...
|
||||
│
|
||||
├── GUI/ # Flask templates (Tailwind + DaisyUI + Lucide)
|
||||
│ ├── layout.html # Base layout (no jQuery, no Bootstrap)
|
||||
│ ├── index.html # Video Library (3 buttons: source / download / copy link)
|
||||
│ ├── backgrounds.html # Background Manager (videos catalog)
|
||||
│ ├── settings.html # Config editor (validated against template)
|
||||
│ └── create.html # Render progress page
|
||||
│
|
||||
├── tests/
|
||||
│ └── test_gui_utils.py # pytest regression for add/delete background
|
||||
│
|
||||
├── main.py # CLI entry (platform-routed via factory)
|
||||
├── GUI.py # Flask web UI; `/video/<id>` serves files with sanitized headers
|
||||
├── Dockerfile # python:3.10-slim-bookworm + ffmpeg + playwright + pytest
|
||||
├── docker-compose.yml # Services: gui, cli, test
|
||||
├── docker-entrypoint.sh # Runs `utils.docker_bootstrap` then exec's the command
|
||||
├── requirements.txt
|
||||
└── CLAUDE.md
|
||||
```
|
||||
|
||||
---
|
||||
|
||||
## Configuration
|
||||
|
||||
### Threads (full config)
|
||||
|
||||
```toml
|
||||
[settings]
|
||||
platform = "threads"
|
||||
|
||||
[threads]
|
||||
discovery_method = "scrape" # "api" (Graph API, own posts) or "scrape" (trending feed)
|
||||
|
||||
[threads.creds]
|
||||
username = "your_insta" # For Playwright login (always needed)
|
||||
password = "your_password"
|
||||
access_token = "" # Only for discovery_method="api"
|
||||
user_id = "" # Only for discovery_method="api"
|
||||
|
||||
[threads.thread]
|
||||
post_id = "" # Specific post ID; blank = auto-pick from feed
|
||||
max_reply_length = 500
|
||||
min_reply_length = 1
|
||||
min_replies = 5 # Minimum replies for post eligibility
|
||||
min_engagement = 0 # Minimum likes+reposts for viral filter (0=disabled, 10000=viral)
|
||||
blocked_words = ""
|
||||
|
||||
[settings.tts]
|
||||
voice_choice = "googletranslate" # Best for macOS: no API key, fast, free
|
||||
# voice_choice = "tiktok" # Needs tiktok_sessionid; auto-falls back to pyttsx3
|
||||
# voice_choice = "OpenAI" # Needs openai_api_key
|
||||
|
||||
[settings.background]
|
||||
background_video = "minecraft"
|
||||
background_audio = "lofi"
|
||||
background_audio_volume = 0.15
|
||||
```
|
||||
|
||||
### Reddit (reference)
|
||||
|
||||
```toml
|
||||
[settings]
|
||||
platform = "reddit"
|
||||
|
||||
[reddit.creds]
|
||||
client_id = "..."
|
||||
client_secret = "..."
|
||||
username = "..."
|
||||
password = "..."
|
||||
2fa = false
|
||||
2fa_secret = "" # TOTP base32 secret for auto-2FA
|
||||
|
||||
[reddit.thread]
|
||||
subreddit = "AskReddit"
|
||||
min_comments = 20
|
||||
```
|
||||
|
||||
### YouTube upload
|
||||
|
||||
```toml
|
||||
[youtube]
|
||||
enabled = false # Set true to auto-upload after render
|
||||
privacy = "public" # or "private", "unlisted"
|
||||
client_secret_path = "" # Path to youtube_client_secret.json
|
||||
```
|
||||
|
||||
---
|
||||
|
||||
## Platform-Specific Knowledge
|
||||
|
||||
### Threads — Web Scraping (discovery_method = "scrape")
|
||||
|
||||
**DOM Structure:**
|
||||
- Threads.net uses **div-based card layout** — NO `<article>` elements
|
||||
- Feed posts: `a[href*="/post/"]` links inside `<div>` cards (class contains `x1a2a7pz`)
|
||||
- Post pages: same structure; main post link appears first, replies follow
|
||||
- Screenshots: Use `a[href*="/post/"]` → ancestor div card, NOT `page.locator("article")`
|
||||
|
||||
**Card Text Format (used by `_parse_card_text()`):**
|
||||
```
|
||||
Line 0: username
|
||||
Line 1: timestamp (e.g., "14h", "1d")
|
||||
Line 2..N: post body text
|
||||
Last 1-4: engagement metrics (likes, replies, reposts, quotes)
|
||||
```
|
||||
|
||||
**Engagement Parsing:**
|
||||
- Numbers can be plain ("266") or abbreviated ("1K", "2.5M")
|
||||
- `likes` = first trailing number, `replies` = second, `reposts` = third
|
||||
- `min_engagement` filters by `likes + reposts` total
|
||||
- Posts sorted by engagement descending before selection
|
||||
|
||||
**Login Flow:**
|
||||
- Threads uses Instagram auth (`threads.net/login`)
|
||||
- Selectors: `input[autocomplete="username"]`, `input[autocomplete="current-password"]`
|
||||
- Button: `get_by_role("button", name="Log in", exact=True).first`
|
||||
- After click: `page.wait_for_url("https://www.threads.net/", timeout=15000)` — event-wait, not fixed delay
|
||||
- Cookies cached at `video_creation/data/cookie-threads.json`
|
||||
- Login logic shared via `platforms/threads/auth.py`
|
||||
|
||||
**API Limitation:**
|
||||
- Graph API v1.0 only accesses YOUR OWN posts — no trending/discovery
|
||||
- Scraping bypasses this — no API token needed
|
||||
|
||||
### Threads — Graph API (discovery_method = "api")
|
||||
|
||||
- Auth: Bearer token, 60-day expiry
|
||||
- Only accesses authenticated user's own threads + replies
|
||||
- Use when you have your own content with replies
|
||||
|
||||
### Reddit
|
||||
|
||||
- **API:** PRAW (Python Reddit API Wrapper)
|
||||
- **Post discovery:** `subreddit.hot(limit=25)` → `get_subreddit_undone()` → fallback to `top(day/hour/month/week/year/all)`
|
||||
- **Screenshot:** Playwright on new.reddit.com
|
||||
- **2FA:** Auto-TOTP via `pyotp` when `2fa_secret` is configured in config.toml
|
||||
|
||||
---
|
||||
|
||||
## Development Guidelines
|
||||
|
||||
### ✅ DO:
|
||||
|
||||
1. **Run everything through Docker** — `docker compose up gui`, `docker compose run --rm cli`, `docker compose run --rm test`
|
||||
2. **Use platform factory** — never import platform modules directly
|
||||
3. **Return standard content_object** from all fetchers
|
||||
4. **Use clean body text** for TTS — parse out username/timestamp metadata
|
||||
5. **Default to `googletranslate` TTS** for headless containers — no API key, fast, free
|
||||
6. **Use `libx264` encoder** — `h264_nvenc` is NVIDIA-only and not available in the slim image
|
||||
7. **Test both Threads discovery methods:** `api` and `scrape`
|
||||
8. **Bind-mount preserves state** — edits to `config.toml`, `results/`, `assets/temp/`, `video_creation/data/`, and `utils/background_*.json` catalogs persist across container runs
|
||||
9. **GUI must bind to `0.0.0.0`** in Docker (enforced via `GUI_HOST=0.0.0.0` env)
|
||||
10. **Use `/video/<id>` to serve renders** — the route looks up the file by id in `videos.json`, sanitizes `Content-Disposition` filename, avoids 404s from literal newlines in titles
|
||||
|
||||
### ❌ DON'T:
|
||||
|
||||
1. **Don't run `python GUI.py` or `python main.py` on the host** — Docker is the only supported path
|
||||
2. **Don't use `<article>` selectors** on Threads.net — DOM is div-based
|
||||
3. **Don't hardcode `h264_nvenc`** — use `libx264` for cross-platform compatibility
|
||||
4. **Don't import platform modules directly** in main.py/utils
|
||||
5. **Don't assume config keys exist** without `.get()` fallback
|
||||
6. **Don't reintroduce jQuery, Bootstrap, or ClipboardJS** — UI is vanilla ES6 + Tailwind + DaisyUI + Lucide
|
||||
7. **Don't write to `utils/backgrounds.json`** — legacy empty file. Use `utils/background_videos.json` and `utils/background_audios.json`
|
||||
|
||||
### 🔒 Security (hardened May 2026)
|
||||
|
||||
1. **No `eval()`** — use `{"int": int, "float": float, "bool": bool, "str": str}` dict dispatch. `utils/settings.py` has module-level `_TYPE_COERCION`.
|
||||
2. **No `os.system()`** — use `subprocess.run([...])` with argument lists. No shell interpretation.
|
||||
3. **No `shell=True`** — removed from all `subprocess.run()` and `Popen()` calls.
|
||||
4. **No bare `except:`** — catch specific exception types. Bare excepts swallow `KeyboardInterrupt` and `SystemExit`.
|
||||
5. **Redact secrets before printing** — `main.py` error handler deep-copies config and masks all credential fields. `GUI.py` redacts API keys/passwords from settings page data. Sensitive fields show as `********`.
|
||||
6. **CSRF protection** — `GUI.py` `@app.before_request` checks `Origin` header on all mutating requests.
|
||||
7. **Security headers** — `X-Content-Type-Options: nosniff`, `X-Frame-Options: DENY` on every response.
|
||||
8. **Flask secret key** — `FLASK_SECRET_KEY` env var, fallback `os.urandom(32)` per startup.
|
||||
9. **Docker non-root** — container runs as `appuser`, not root.
|
||||
10. **Path traversal** — `/video/<id>` uses `Path.resolve().relative_to()` guard; `add_background()` sanitizes citation with `re.sub(r"[./\\\\]", "_", citation)`.
|
||||
11. **No hardcoded credentials** in source — all secrets from `config.toml` (gitignored). Rotate passwords regularly.
|
||||
|
||||
---
|
||||
|
||||
## Web UI (Flask, served by `gui` service)
|
||||
|
||||
- **Stack:** Tailwind CSS, DaisyUI, Lucide Icons, vanilla ES6 (no jQuery, Bootstrap, ClipboardJS)
|
||||
- **Routes:**
|
||||
- `/` — Video Library; cards show source-post link, download, copy-link buttons
|
||||
- `/video/<id>` — serves rendered mp4 by id (lookup via `videos.json`); path-traversal guard, sanitized `Content-Disposition`
|
||||
- `/backgrounds` — Background Manager UI
|
||||
- `/backgrounds.json` — serves `utils/background_videos.json` (videos catalog)
|
||||
- `/background/add`, `/background/delete` — POST; mutate **both** `utils/background_videos.json` and `settings.background.background_video.options` in `utils/.config.template.toml`
|
||||
- `/settings` — config editor; loads from `config.toml`, validates against `utils/.config.template.toml`, persists via `utils/gui_utils.modify_settings` (preserves comments/formatting via `tomlkit`)
|
||||
- **HTML escaping:** `h()` helper in `index.html` escapes `& " < >` for user-controlled strings in attributes
|
||||
|
||||
---
|
||||
|
||||
## Key Files to Know
|
||||
|
||||
| File | Purpose |
|
||||
|------|---------|
|
||||
| `main.py` | CLI entry; pipeline orchestration via factory |
|
||||
| `platforms/__init__.py` | Factory dispatch (platform + discovery_method) |
|
||||
| `platforms/threads/scraper.py` | Web scraping fetcher with engagement parsing |
|
||||
| `platforms/threads/auth.py` | Shared Playwright login + cookie management |
|
||||
| `platforms/threads/fetcher.py` | Graph API client (own posts only) |
|
||||
| `platforms/threads/screenshot.py` | Div-based Threads screenshotter |
|
||||
| `video_creation/final_video.py` | FFmpeg composition (libx264, platform-aware output); exports `get_output_path()` |
|
||||
| `video_creation/background.py` | Background downloader (local files + yt-dlp); prefers already-downloaded videos |
|
||||
| `video_creation/youtube_uploader.py` | OAuth2 YouTube upload |
|
||||
| `TTS/engine_wrapper.py` | TTS provider abstraction + TikTok→pyttsx3 fallback; single-pass ffmpeg concat |
|
||||
| `TTS/TikTok.py` | Hardened TikTok TTS with graceful error handling |
|
||||
| `reddit/subreddit.py` | PRAW Reddit fetcher with auto-2FA; retry-depth limit (50) |
|
||||
| `utils/settings.py` | Config loading + interactive validation; uses `_TYPE_COERCION` dict (no eval) |
|
||||
| `utils/videos.py` | Video dedup tracking (`check_done`, `check_done_by_id`, `save_data` with truncate) |
|
||||
| `utils/.config.template.toml` | Config schema (drives Settings page validation) |
|
||||
| `utils/background_videos.json` | Background video manifest (served at `/backgrounds.json`) |
|
||||
| `utils/background_audios.json` | Background audio manifest |
|
||||
| `utils/gui_utils.py` | `add_background`, `delete_background`, `modify_settings`, `get_checks` (no eval) |
|
||||
| `GUI.py` | Flask app: `/`, `/video/<id>`, `/backgrounds`, `/settings`, `/create`; CSRF + security headers |
|
||||
| `Dockerfile` | python:3.14-slim-bookworm + ffmpeg + Playwright Chromium + pytest; runs as `appuser` |
|
||||
| `docker-compose.yml` | Three services: `gui` (port 4000), `cli`, `test` |
|
||||
| `tests/test_gui_utils.py` | Pytest regression for Background Manager round-trip |
|
||||
|
||||
---
|
||||
|
||||
## Debugging Tips
|
||||
|
||||
### FFmpeg "Unknown encoder 'h264_nvenc'"
|
||||
→ Use `libx264`. Find-and-replace `h264_nvenc` → `libx264` in `video_creation/final_video.py`. Slim image doesn't ship NVIDIA encoders.
|
||||
|
||||
### yt-dlp "Requested format is not available"
|
||||
→ Bump pinned version in `requirements.txt` and rebuild (`docker compose build`). Prefer `best[height<=1080]` over `bestvideo` in `video_creation/background.py` — many videos lack video-only streams.
|
||||
|
||||
### Threads screenshots fail ("Main post article not found")
|
||||
→ Threads.net uses div cards, not `<article>`. Use `a[href*="/post/"]` → ancestor div approach.
|
||||
|
||||
### Config validator EOFError in non-interactive mode
|
||||
→ `check_toml()` prompts for ALL platform sections regardless of `platform` setting. Fill all required fields, edit through `/settings`, or pre-populate `config.toml` before `docker compose run cli`.
|
||||
|
||||
### Playwright timeout on Threads login
|
||||
→ Cookies corrupted. Delete `video_creation/data/cookie-threads.json` for fresh login (file is bind-mounted, host delete clears container too). Confirm selectors: button uses `exact=True` for multiple "Log in" buttons.
|
||||
|
||||
### No viral posts found
|
||||
→ Lower `min_engagement` in config. Most Threads feed posts have <100 likes — 10000 filters almost everything.
|
||||
|
||||
### Background Manager grid is empty
|
||||
→ `/backgrounds.json` must serve `utils/background_videos.json` (split catalog), **not** legacy `utils/backgrounds.json` (empty `{}`). Verify in `GUI.py:backgrounds_json`.
|
||||
|
||||
### `/video/<id>` returns 404
|
||||
→ Route looks up entry in `video_creation/data/videos.json` by `id`, resolves file under `results/<thread_category>/<filename>.mp4`. Confirm both JSON entry and file exist; file may have been pruned.
|
||||
|
||||
### JS "Unexpected end of input" on Library page
|
||||
→ User-controlled strings in HTML attributes must go through `h()` helper in `index.html`. Avoid inline `onclick=` with `${JSON.stringify(...)}`.
|
||||
|
||||
### Stale image after editing `requirements.txt` or `Dockerfile`
|
||||
→ `docker compose build` to rebuild. Code-only changes don't need rebuild — repo root is bind-mounted to `/app`.
|
||||
|
||||
### Python bytecode caching in long-running GUI container
|
||||
→ GUI caches imported modules in `sys.modules`. After editing pipeline code, restart GUI (`docker compose restart gui`) or trigger pipeline run which calls `importlib.reload()` on pipeline modules.
|
||||
|
||||
### Reddit image template appearing in Threads videos
|
||||
→ Verify `platform` in config.toml is `"threads"` (not `"reddit"`). The `if platform == "reddit"` guard in `final_video.py` blocks Reddit template. Restart GUI container to flush Python bytecode cache.
|
||||
|
||||
### Background video download fails (yt-dlp HTTP 403)
|
||||
→ `get_background_config()` prefers already-downloaded videos. Set `background_video` in config.toml to a downloaded video name (check `assets/backgrounds/video/`). If empty, randomly picks from downloaded videos first.
|
||||
|
||||
### TTS output has wrong number of audio clips
|
||||
→ `engine_wrapper.run()` returns `idx + 1` (count, not last index). If getting one fewer clip than expected, check return value consumers — treat as count.
|
||||
|
||||
### videos.json corruption (trailing garbage after save)
|
||||
→ Fixed: `save_data()` calls `raw_vids.truncate()` after `json.dump()`. Delete `video_creation/data/videos.json` if existing file is corrupted.
|
||||
|
||||
### Infinite recursion in Reddit post discovery
|
||||
→ Fixed: `get_subreddit_threads()` has retry-depth limit of 50. If hit, subreddit may have no undone posts — try different subreddit or clear `videos.json`.
|
||||
|
||||
---
|
||||
|
||||
## Useful Commands (Docker-only)
|
||||
|
||||
```bash
|
||||
# Build (or rebuild after Dockerfile / requirements.txt changes)
|
||||
docker compose build
|
||||
|
||||
# Run the GUI (foreground)
|
||||
docker compose up gui
|
||||
# → http://localhost:4000
|
||||
|
||||
# Run the GUI in background
|
||||
docker compose up -d gui
|
||||
docker compose logs -f gui
|
||||
docker compose down
|
||||
|
||||
# Run CLI pipeline (one-off, removed on exit)
|
||||
docker compose run --rm cli
|
||||
docker compose run --rm cli python main.py <post_id>
|
||||
|
||||
# Run test suite
|
||||
docker compose run --rm test
|
||||
|
||||
# Shell in fresh container for ad-hoc commands
|
||||
docker compose run --rm --entrypoint /bin/bash gui
|
||||
# inside: python -m py_compile main.py platforms/threads/scraper.py
|
||||
|
||||
# Tail running GUI container
|
||||
docker compose exec gui ls /app/results/threads/
|
||||
```
|
||||
|
||||
> Anything needing `pip install`, `playwright install`, or `apt-get` belongs in `Dockerfile` + `docker compose build` — never on host.
|
||||
|
||||
---
|
||||
|
||||
## Recent Changes (May 2026 Security Hardening)
|
||||
|
||||
**eval() removal:** `eval(checks["type"])(value)` replaced with `{"int": int, "float": float, "bool": bool, "str": str}` dict dispatch in `utils/settings.py`, `utils/console.py`, `utils/gui_utils.py`.
|
||||
|
||||
**os.system() removal:** `TTS/engine_wrapper.py:split_post` uses `subprocess.run([...])` with argument lists. `utils/posttextparser.py` spacy download uses `subprocess.run([sys.executable, "-m", "spacy", ...])`.
|
||||
|
||||
**shell=True removal:** All `subprocess.run(..., shell=True)` and `Popen(..., shell=True)` replaced with argument lists in `main.py` and `utils/ffmpeg_install.py`.
|
||||
|
||||
**Credential leak prevention:** `main.py` error handler deep-copies config and redacts all secrets. `GUI.py` masks sensitive keys as `********` in settings page data.
|
||||
|
||||
**CSRF + security headers:** `GUI.py` checks `Origin` header on POST/PUT/DELETE. `X-Content-Type-Options`, `X-Frame-Options` headers added.
|
||||
|
||||
**Docker hardening:** Container runs as `appuser` (non-root). Digest pinning + pip version comments added.
|
||||
|
||||
**Bug fixes (18 total):**
|
||||
- Config overwrite crash (config=None after empty file write)
|
||||
- Playwright TimeoutError (wrong exception class caught)
|
||||
- Lambda closure (loop variable captured by reference)
|
||||
- Redundant ffmpeg runs (concat now single-pass)
|
||||
- Audio IndexError on empty TTS output
|
||||
- Hardcoded NSFW post selector (now generic role-based)
|
||||
- JSON truncation bug in save_data (missing truncate())
|
||||
- Infinite recursion in Reddit post discovery (retry limit 50)
|
||||
- Silent exception swallowing in scraper search
|
||||
- exit() → sys.exit() in subreddit.py
|
||||
- Dead macOS branch (os.name == "mac" → sys.platform == "darwin")
|
||||
- Wrong upstream repo in version check (now configurable + resilient)
|
||||
- Duplicate path logic (get_output_path() shared between main.py and final_video.py)
|
||||
- Catastrophic backtracking URL regex (now atomic https?://\S+)
|
||||
- Fixed 6s login delay (now wait_for_url event-wait)
|
||||
- 6 bare except: clauses → specific exception types
|
||||
- Temp file leak in ProgressFfmpeg (cleanup in __exit__)
|
||||
- Flask secret key hardcoded → env var + urandom fallback
|
||||
@ -1,12 +1,51 @@
|
||||
FROM python:3.10.14-slim
|
||||
FROM python:3.14-slim-bookworm
|
||||
|
||||
RUN apt update
|
||||
RUN apt-get install -y ffmpeg
|
||||
RUN apt install python3-pip -y
|
||||
ENV PYTHONDONTWRITEBYTECODE=1 \
|
||||
PYTHONUNBUFFERED=1 \
|
||||
PIP_NO_CACHE_DIR=1 \
|
||||
PLAYWRIGHT_BROWSERS_PATH=/ms-playwright \
|
||||
XDG_CACHE_HOME=/app/.cache
|
||||
|
||||
RUN mkdir /app
|
||||
ADD . /app
|
||||
WORKDIR /app
|
||||
RUN pip install -r requirements.txt
|
||||
|
||||
CMD ["python3", "main.py"]
|
||||
# System deps: ffmpeg for video processing
|
||||
RUN apt-get update \
|
||||
&& apt-get install -y --no-install-recommends \
|
||||
ffmpeg \
|
||||
curl \
|
||||
&& rm -rf /var/lib/apt/lists/*
|
||||
|
||||
# Install PyTorch CPU-only first (saves ~1.5GB vs GPU version)
|
||||
RUN pip install --no-cache-dir torch==2.11.0 --index-url https://download.pytorch.org/whl/cpu
|
||||
|
||||
# Install remaining Python deps
|
||||
COPY requirements.txt ./
|
||||
RUN pip install --no-cache-dir --upgrade pip \
|
||||
&& pip install --no-cache-dir -r requirements.txt
|
||||
|
||||
# Install Chromium for Playwright (already cached at /ms-playwright)
|
||||
RUN python -m playwright install --with-deps chromium
|
||||
|
||||
# Clean up pip cache and temp files
|
||||
RUN pip cache purge 2>/dev/null || true \
|
||||
&& find /usr/local/lib/python3.14 -name '__pycache__' -exec rm -rf {} + 2>/dev/null || true \
|
||||
&& find /usr/local/lib/python3.14 -name '*.pyc' -delete 2>/dev/null || true \
|
||||
&& rm -rf /root/.cache
|
||||
|
||||
# Copy app code
|
||||
COPY . .
|
||||
|
||||
# Create non-root user
|
||||
RUN groupadd -r appuser && useradd -r -g appuser -d /app appuser \
|
||||
&& chown -R appuser:appuser /app /ms-playwright
|
||||
|
||||
ENV CLOAKBROWSER_CACHE_DIR=/app/.cache/cloakbrowser
|
||||
ENV PUBLIC_BASE_PATH=/threads-video-maker
|
||||
ENV PUBLIC_DEMO_MODE=1
|
||||
|
||||
RUN chmod +x /app/docker-entrypoint.sh
|
||||
|
||||
USER appuser
|
||||
|
||||
ENTRYPOINT ["/bin/sh", "/app/docker-entrypoint.sh"]
|
||||
CMD ["python", "GUI.py"]
|
||||
|
||||
@ -1,263 +1,244 @@
|
||||
{% extends "layout.html" %}
|
||||
{% block main %}
|
||||
|
||||
<!-- Delete Background Modal -->
|
||||
<div class="modal fade" id="deleteBtnModal" tabindex="-1" role="dialog" aria-hidden="true">
|
||||
<div class="modal-dialog modal-dialog-centered" role="document">
|
||||
<div class="modal-content">
|
||||
<div class="modal-header">
|
||||
<h5 class="modal-title">Delete background</h5>
|
||||
</div>
|
||||
<div class="modal-body">
|
||||
Are you sure you want to delete this background?
|
||||
</div>
|
||||
<div class="modal-footer">
|
||||
<button type="button" class="btn btn-secondary" data-dismiss="modal">Close</button>
|
||||
<form action="background/delete" method="post">
|
||||
<input type="hidden" id="background-key" name="background-key" value="">
|
||||
<button type="submit" class="btn btn-danger">Delete</button>
|
||||
</form>
|
||||
<div class="bg-[#F2EFEB] min-h-[calc(100vh-6.5rem)] py-8">
|
||||
<div class="container mx-auto px-4">
|
||||
<!-- Header & Actions -->
|
||||
<div class="flex flex-col md:flex-row justify-between items-center gap-4 mb-8">
|
||||
<h1 class="text-3xl font-display font-black uppercase tracking-tighter text-[#111111]">Background Manager</h1>
|
||||
<div class="flex w-full md:w-auto gap-2">
|
||||
<div class="relative flex-grow md:w-64">
|
||||
<i data-lucide="search" class="absolute left-3 top-1/2 -translate-y-1/2 w-4 h-4 text-[#111111]/40"></i>
|
||||
<input type="text"
|
||||
class="searchFilter input-neo w-full pl-10"
|
||||
placeholder="Search..."
|
||||
onkeyup="searchFilter()">
|
||||
</div>
|
||||
<button onclick="add_modal.showModal()" class="btn-primary-neo text-sm" data-demo-disabled>
|
||||
<i data-lucide="plus" class="w-4 h-4"></i>
|
||||
<span class="hidden sm:inline ml-1">Add Video</span>
|
||||
</button>
|
||||
</div>
|
||||
</div>
|
||||
|
||||
<!-- Background Grid -->
|
||||
<div class="grid grid-cols-1 sm:grid-cols-2 lg:grid-cols-3 xl:grid-cols-4 gap-6" id="backgrounds">
|
||||
<!-- Backgrounds will be injected here -->
|
||||
</div>
|
||||
|
||||
<!-- Empty State -->
|
||||
<div id="empty-state" class="hidden flex flex-col items-center justify-center py-20 text-[#111111]/30">
|
||||
<i data-lucide="film" class="w-16 h-16 mb-4"></i>
|
||||
<p class="text-lg font-mono uppercase tracking-wider">No backgrounds found</p>
|
||||
</div>
|
||||
</div>
|
||||
</div>
|
||||
|
||||
<!-- Delete Background Modal -->
|
||||
<dialog id="delete_modal" class="modal modal-bottom sm:modal-middle">
|
||||
<div class="modal-box bg-white border border-[#111111]">
|
||||
<h3 class="font-display font-black uppercase tracking-tighter text-lg text-[#111111]">Delete Background</h3>
|
||||
<p class="py-4 text-[#111111]/60 font-mono text-sm">Are you sure you want to delete this background video? This action cannot be undone.</p>
|
||||
<div class="modal-action">
|
||||
<form action="background/delete" method="post" class="flex gap-2">
|
||||
<input type="hidden" id="background-key" name="background-key" value="">
|
||||
<button type="button" onclick="delete_modal.close()" class="btn-ghost-neo">Cancel</button>
|
||||
<button type="submit" class="btn-danger-neo text-sm" data-demo-disabled>Delete</button>
|
||||
</form>
|
||||
</div>
|
||||
</div>
|
||||
</dialog>
|
||||
|
||||
<!-- Add Background Modal -->
|
||||
<div class="modal fade" id="backgroundAddModal" tabindex="-1" role="dialog" aria-hidden="true">
|
||||
<div class="modal-dialog modal-dialog-centered" role="document">
|
||||
<div class="modal-content">
|
||||
<div class="modal-header">
|
||||
<h5 class="modal-title">Add background video</h5>
|
||||
<dialog id="add_modal" class="modal modal-bottom sm:modal-middle">
|
||||
<div class="modal-box bg-white border border-[#111111] max-w-lg">
|
||||
<h3 class="font-display font-black uppercase tracking-tighter text-lg text-[#111111] mb-6">Add Background Video</h3>
|
||||
<form id="addBgForm" action="background/add" method="post" novalidate class="space-y-4">
|
||||
<div class="form-control w-full">
|
||||
<label class="label"><span class="label-text text-[#111111]/70 font-mono text-xs uppercase tracking-wider">YouTube URI</span></label>
|
||||
<div class="flex border border-[#111111]">
|
||||
<div class="flex items-center justify-center px-3 bg-[#111111]/5 border-r border-[#111111] shrink-0">
|
||||
<i data-lucide="youtube" class="w-4 h-4 text-[#111111]"></i>
|
||||
</div>
|
||||
<input name="youtube_uri" type="text" placeholder="https://www.youtube.com/watch?v=..."
|
||||
class="input-neo border-0 flex-1">
|
||||
</div>
|
||||
<label class="label h-6"><span id="feedbackYT" class="label-text-alt text-[#DE6C56] font-mono text-xs hidden"></span></label>
|
||||
</div>
|
||||
<div class="modal-body">
|
||||
|
||||
<!-- Add video form -->
|
||||
<form id="addBgForm" action="background/add" method="post" novalidate>
|
||||
<div class="form-group row">
|
||||
<label class="col-4 col-form-label" for="youtube_uri">YouTube URI</label>
|
||||
<div class="col-8">
|
||||
<div class="input-group">
|
||||
<div class="input-group-text">
|
||||
<i class="bi bi-youtube"></i>
|
||||
</div>
|
||||
<input name="youtube_uri" placeholder="https://www.youtube.com/watch?v=..." type="text"
|
||||
class="form-control">
|
||||
</div>
|
||||
<span id="feedbackYT" class="form-text feedback-invalid"></span>
|
||||
</div>
|
||||
<div class="form-control w-full">
|
||||
<label class="label"><span class="label-text text-[#111111]/70 font-mono text-xs uppercase tracking-wider">Filename</span></label>
|
||||
<div class="flex border border-[#111111]">
|
||||
<div class="flex items-center justify-center px-3 bg-[#111111]/5 border-r border-[#111111] shrink-0">
|
||||
<i data-lucide="file-video" class="w-4 h-4 text-[#111111]"></i>
|
||||
</div>
|
||||
<div class="form-group row">
|
||||
<label for="filename" class="col-4 col-form-label">Filename</label>
|
||||
<div class="col-8">
|
||||
<div class="input-group">
|
||||
<div class="input-group-text">
|
||||
<i class="bi bi-file-earmark"></i>
|
||||
</div>
|
||||
<input name="filename" placeholder="Example: cool-background" type="text"
|
||||
class="form-control">
|
||||
</div>
|
||||
<span id="feedbackFilename" class="form-text feedback-invalid"></span>
|
||||
</div>
|
||||
</div>
|
||||
<div class="form-group row">
|
||||
<label for="citation" class="col-4 col-form-label">Credits (owner of the video)</label>
|
||||
<div class="col-8">
|
||||
<div class="input-group">
|
||||
<div class="input-group-text">
|
||||
<i class="bi bi-person-circle"></i>
|
||||
</div>
|
||||
<input name="citation" placeholder="YouTube Channel" type="text" class="form-control">
|
||||
</div>
|
||||
<span class="form-text text-muted">Include the channel name of the
|
||||
owner of the background video you are adding.</span>
|
||||
</div>
|
||||
</div>
|
||||
<div class="form-group row">
|
||||
<label for="position" class="col-4 col-form-label">Position of screenshots</label>
|
||||
<div class="col-8">
|
||||
<div class="input-group">
|
||||
<div class="input-group-text">
|
||||
<i class="bi bi-arrows-fullscreen"></i>
|
||||
</div>
|
||||
<input name="position" placeholder="Example: center" type="text" class="form-control">
|
||||
</div>
|
||||
<span class="form-text text-muted">Advanced option (you can leave it
|
||||
empty). Valid options are "center" and decimal numbers</span>
|
||||
</div>
|
||||
</div>
|
||||
</div>
|
||||
<div class="modal-footer">
|
||||
<button type="button" class="btn btn-secondary" data-dismiss="modal">Close</button>
|
||||
<button name="submit" type="submit" class="btn btn-success">Add background</button>
|
||||
</form>
|
||||
<input name="filename" type="text" placeholder="e.g. minecraft-parkour"
|
||||
class="input-neo border-0 flex-1">
|
||||
</div>
|
||||
<label class="label h-6"><span id="feedbackFilename" class="label-text-alt text-[#DE6C56] font-mono text-xs hidden"></span></label>
|
||||
</div>
|
||||
</div>
|
||||
</div>
|
||||
</div>
|
||||
|
||||
<main>
|
||||
<div class="album py-2 bg-light">
|
||||
<div class="container">
|
||||
|
||||
<div class="row justify-content-between mt-2">
|
||||
<div class="col-12 col-md-3 mb-3">
|
||||
<input type="text" class="form-control searchFilter" placeholder="Search backgrounds"
|
||||
onkeyup="searchFilter()">
|
||||
</div>
|
||||
<div class="col-12 col-md-2 mb-3">
|
||||
<button type="button" class="btn btn-primary form-control" data-toggle="modal"
|
||||
data-target="#backgroundAddModal">
|
||||
Add background video
|
||||
</button>
|
||||
<div class="form-control w-full">
|
||||
<label class="label"><span class="label-text text-[#111111]/70 font-mono text-xs uppercase tracking-wider">Credits</span></label>
|
||||
<div class="flex border border-[#111111]">
|
||||
<div class="flex items-center justify-center px-3 bg-[#111111]/5 border-r border-[#111111] shrink-0">
|
||||
<i data-lucide="user" class="w-4 h-4 text-[#111111]"></i>
|
||||
</div>
|
||||
<input name="citation" type="text" placeholder="YouTube Channel Name"
|
||||
class="input-neo border-0 flex-1">
|
||||
</div>
|
||||
<label class="label"><span class="label-text-alt text-[#111111]/40 font-mono text-xs">Name of the video owner.</span></label>
|
||||
</div>
|
||||
|
||||
<div class="grid row row-cols-1 row-cols-sm-2 row-cols-md-3 g-3" id="backgrounds">
|
||||
<div class="form-control w-full">
|
||||
<label class="label"><span class="label-text text-[#111111]/70 font-mono text-xs uppercase tracking-wider">Advanced: Position</span></label>
|
||||
<input name="position" type="text" placeholder="center (optional)"
|
||||
class="input-neo w-full text-sm h-10">
|
||||
</div>
|
||||
|
||||
<div class="modal-action">
|
||||
<button type="button" onclick="add_modal.close()" class="btn-ghost-neo">Cancel</button>
|
||||
<button type="submit" class="btn-primary-neo text-sm" data-demo-disabled>Add Background</button>
|
||||
</div>
|
||||
</div>
|
||||
</form>
|
||||
</div>
|
||||
</main>
|
||||
</dialog>
|
||||
|
||||
<script>
|
||||
var keys = [];
|
||||
var youtube_urls = [];
|
||||
let keys = [];
|
||||
let youtube_urls = [];
|
||||
|
||||
function h(str) {
|
||||
return String(str ?? '')
|
||||
.replace(/&/g, '&')
|
||||
.replace(/"/g, '"')
|
||||
.replace(/'/g, ''')
|
||||
.replace(/</g, '<')
|
||||
.replace(/>/g, '>');
|
||||
}
|
||||
|
||||
// Show background videos
|
||||
$(document).ready(function () {
|
||||
$.getJSON("backgrounds.json",
|
||||
function (data) {
|
||||
delete data["__comment"];
|
||||
var background = '';
|
||||
$.each(data, function (key, value) {
|
||||
// Add YT urls and keys (for validation)
|
||||
keys.push(key);
|
||||
youtube_urls.push(value[0]);
|
||||
async function loadBackgrounds() {
|
||||
try {
|
||||
const response = await fetch("backgrounds.json");
|
||||
const data = await response.json();
|
||||
delete data["__comment"];
|
||||
|
||||
const container = document.getElementById('backgrounds');
|
||||
let html = '';
|
||||
|
||||
Object.entries(data).forEach(([key, value]) => {
|
||||
keys.push(key);
|
||||
youtube_urls.push(value[0]);
|
||||
|
||||
const videoId = value[0].includes('?v=') ? value[0].split('?v=')[1] : value[0].split('/').pop();
|
||||
|
||||
html += `
|
||||
<div class="bg-card group bg-white border border-[#111111] hover:border-[#DE6C56] transition-colors duration-200">
|
||||
<div class="aspect-video w-full bg-black border-b border-[#111111] relative">
|
||||
<iframe class="w-full h-full"
|
||||
src="https://www.youtube-nocookie.com/embed/${videoId}"
|
||||
title="YouTube video player"
|
||||
frameborder="0"
|
||||
allow="accelerometer; autoplay; clipboard-write; encrypted-media; gyroscope; picture-in-picture"
|
||||
allowfullscreen></iframe>
|
||||
</div>
|
||||
<div class="p-4">
|
||||
<h3 class="text-[#111111] font-mono text-sm font-medium truncate mb-1" title="${h(key)}">${h(key)}</h3>
|
||||
<p class="text-[#111111]/40 font-mono text-xs truncate mb-4">${h(value[2])}</p>
|
||||
|
||||
${window.PUBLIC_DEMO_MODE ? '' : `<div class="flex justify-end">
|
||||
<button onclick="confirmDelete('${key}')" class="btn-ghost-neo p-2 hover:text-[#DE6C56]">
|
||||
<i data-lucide="trash-2" class="w-4 h-4"></i>
|
||||
</button>
|
||||
</div>`}
|
||||
</div>
|
||||
</div>
|
||||
`;
|
||||
});
|
||||
|
||||
background += '<div class="col">';
|
||||
background += '<div class="card shadow-sm">';
|
||||
background += '<iframe class="bd-placeholder-img card-img-top" width="100%" height="225" src="https://www.youtube-nocookie.com/embed/' + value[0].split("?v=")[1] + '" title="YouTube video player" frameborder="0" allow="accelerometer; autoplay; clipboard-write; encrypted-media; gyroscope; picture-in-picture" allowfullscreen></iframe>';
|
||||
background += '<div class="card-body">';
|
||||
background += '<p class="card-text">' + value[2] + ' • ' + key + '</p>';
|
||||
background += '<div class="d-flex justify-content-between align-items-center">';
|
||||
background += '<div class="btn-group">';
|
||||
background += '<button type="button" class="btn btn-outline-danger" data-toggle="modal" data-target="#deleteBtnModal" data-background-key="' + key + '">Delete</button>';
|
||||
background += '</div>';
|
||||
background += '</div>';
|
||||
background += '</div>';
|
||||
background += '</div>';
|
||||
background += '</div>';
|
||||
});
|
||||
container.innerHTML = html;
|
||||
lucide.createIcons();
|
||||
} catch (error) {
|
||||
console.error("Error loading backgrounds:", error);
|
||||
}
|
||||
}
|
||||
|
||||
$('#backgrounds').append(background);
|
||||
});
|
||||
});
|
||||
function confirmDelete(key) {
|
||||
document.getElementById('background-key').value = key;
|
||||
delete_modal.showModal();
|
||||
}
|
||||
|
||||
// Add background key when deleting
|
||||
$('#deleteBtnModal').on('show.bs.modal', function (event) {
|
||||
var button = $(event.relatedTarget);
|
||||
var key = button.data('background-key');
|
||||
function searchFilter() {
|
||||
const query = document.querySelector(".searchFilter") ? document.querySelector(".searchFilter").value.toLowerCase() : '';
|
||||
const cards = document.querySelectorAll(".bg-card");
|
||||
let visibleCount = 0;
|
||||
|
||||
$('#background-key').prop('value', key);
|
||||
});
|
||||
cards.forEach(card => {
|
||||
const text = card.textContent.toLowerCase();
|
||||
const matches = text.includes(query);
|
||||
card.classList.toggle('hidden', !matches);
|
||||
if (matches) visibleCount++;
|
||||
});
|
||||
|
||||
var searchFilter = () => {
|
||||
const input = document.querySelector(".searchFilter");
|
||||
const cards = document.getElementsByClassName("col");
|
||||
console.log(cards[1])
|
||||
let filter = input.value
|
||||
for (let i = 0; i < cards.length; i++) {
|
||||
let title = cards[i].querySelector(".card-text");
|
||||
if (title.innerText.toLowerCase().indexOf(filter.toLowerCase()) > -1) {
|
||||
cards[i].classList.remove("d-none")
|
||||
} else {
|
||||
cards[i].classList.add("d-none")
|
||||
}
|
||||
}
|
||||
document.getElementById('empty-state').classList.toggle('hidden', visibleCount > 0);
|
||||
}
|
||||
|
||||
// Validate form
|
||||
$("#addBgForm").submit(function (event) {
|
||||
$("#addBgForm input").each(function () {
|
||||
if (!(validate($(this)))) {
|
||||
event.preventDefault();
|
||||
event.stopPropagation();
|
||||
}
|
||||
const form = document.getElementById('addBgForm');
|
||||
form.addEventListener('submit', (e) => {
|
||||
let isValid = true;
|
||||
form.querySelectorAll('input').forEach(input => {
|
||||
if (!validate(input)) isValid = false;
|
||||
});
|
||||
if (!isValid) e.preventDefault();
|
||||
});
|
||||
|
||||
$('#addBgForm input[type="text"]').on("keyup", function () {
|
||||
validate($(this));
|
||||
form.querySelectorAll('input').forEach(input => {
|
||||
input.addEventListener('keyup', () => validate(input));
|
||||
});
|
||||
|
||||
function validate(object) {
|
||||
let bool = check(object.prop("name"), object.prop("value"));
|
||||
|
||||
// Change class
|
||||
if (bool) {
|
||||
object.removeClass("is-invalid");
|
||||
object.addClass("is-valid");
|
||||
}
|
||||
else {
|
||||
object.removeClass("is-valid");
|
||||
object.addClass("is-invalid");
|
||||
}
|
||||
|
||||
return bool;
|
||||
|
||||
// Check values (return true/false)
|
||||
function check(name, value) {
|
||||
if (name == "youtube_uri") {
|
||||
// URI validation
|
||||
let regex = /(?:\/|%3D|v=|vi=)([0-9A-z-_]{11})(?:[%#?&]|$)/;
|
||||
if (!(regex.test(value))) {
|
||||
$("#feedbackYT").html("Invalid URI");
|
||||
$("#feedbackYT").show();
|
||||
return false;
|
||||
}
|
||||
|
||||
// Check if this background already exists
|
||||
if (youtube_urls.includes(value)) {
|
||||
$("#feedbackYT").html("This background is already added");
|
||||
$("#feedbackYT").show();
|
||||
return false;
|
||||
}
|
||||
|
||||
$("#feedbackYT").hide();
|
||||
return true;
|
||||
}
|
||||
|
||||
if (name == "filename") {
|
||||
// Check if key is already taken
|
||||
if (keys.includes(value)) {
|
||||
$("#feedbackFilename").html("This filename is already taken");
|
||||
$("#feedbackFilename").show();
|
||||
return false;
|
||||
}
|
||||
|
||||
let regex = /^([a-zA-Z0-9\s_-]{1,100})$/;
|
||||
if (!(regex.test(value))) {
|
||||
return false;
|
||||
}
|
||||
|
||||
return true;
|
||||
function validate(input) {
|
||||
const name = input.name;
|
||||
const value = input.value;
|
||||
let valid = true;
|
||||
let message = "";
|
||||
|
||||
if (name === "youtube_uri") {
|
||||
const regex = /(?:\/|%3D|v=|vi=)([0-9A-Za-z_-]{11})(?:[%#?&]|$)/;
|
||||
if (!regex.test(value)) {
|
||||
message = "Invalid YouTube URI";
|
||||
valid = false;
|
||||
} else if (youtube_urls.includes(value)) {
|
||||
message = "Background already added";
|
||||
valid = false;
|
||||
}
|
||||
const feedback = document.getElementById('feedbackYT');
|
||||
feedback.textContent = message;
|
||||
feedback.classList.toggle('hidden', valid);
|
||||
}
|
||||
|
||||
if (name == "citation") {
|
||||
if (value.trim()) {
|
||||
return true;
|
||||
}
|
||||
if (name === "filename") {
|
||||
if (keys.includes(value)) {
|
||||
message = "Filename already taken";
|
||||
valid = false;
|
||||
} else if (!/^([a-zA-Z0-9\s_-]{1,100})$/.test(value)) {
|
||||
valid = false;
|
||||
}
|
||||
const feedback = document.getElementById('feedbackFilename');
|
||||
feedback.textContent = message;
|
||||
feedback.classList.toggle('hidden', valid);
|
||||
}
|
||||
|
||||
if (name == "position") {
|
||||
if (!(value == "center" || value.length == 0 || value % 1 == 0)) {
|
||||
return false;
|
||||
}
|
||||
|
||||
return true;
|
||||
if (name === "position") {
|
||||
if (value && value !== "center" && isNaN(parseFloat(value))) {
|
||||
valid = false;
|
||||
}
|
||||
}
|
||||
|
||||
input.classList.toggle('input-error', !valid);
|
||||
return valid;
|
||||
}
|
||||
|
||||
document.addEventListener('DOMContentLoaded', loadBackgrounds);
|
||||
</script>
|
||||
|
||||
{% endblock %}
|
||||
{% endblock %}
|
||||
|
||||
@ -0,0 +1,571 @@
|
||||
{% extends "layout.html" %}
|
||||
{% block main %}
|
||||
|
||||
<div class="bg-[#F2EFEB] min-h-[calc(100vh-6.5rem)] py-12">
|
||||
<div class="container mx-auto px-4">
|
||||
<div class="grid grid-cols-1 lg:grid-cols-5 gap-6">
|
||||
|
||||
<!-- LEFT PANEL: Controls + Progress -->
|
||||
<div class="lg:col-span-2">
|
||||
<div class="card-neo">
|
||||
<div class="p-8">
|
||||
<div class="flex items-center gap-4 mb-8">
|
||||
<div class="bg-[#DE6C56]/10 p-3 border border-[#DE6C56]">
|
||||
<i data-lucide="plus-square" class="w-8 h-8 text-[#DE6C56]"></i>
|
||||
</div>
|
||||
<div>
|
||||
<h2 class="font-display font-black uppercase tracking-tighter text-2xl text-[#111111]">Create New Short</h2>
|
||||
<p class="text-[#111111]/50 font-mono text-xs mt-1">Start the automated video creation pipeline.</p>
|
||||
</div>
|
||||
</div>
|
||||
|
||||
<div class="space-y-6">
|
||||
<!-- Search Keywords Input -->
|
||||
<div class="form-control w-full">
|
||||
<label class="label px-0">
|
||||
<span class="label-text text-[#111111] font-mono text-xs uppercase tracking-wider font-medium">Search Keywords</span>
|
||||
<span class="label-text-alt text-[#111111]/40 font-mono text-[10px]">Optional</span>
|
||||
</label>
|
||||
<div class="flex gap-2">
|
||||
<input id="keywords-input" type="text"
|
||||
class="input-neo flex-1"
|
||||
placeholder="news, politics, trending, viral"
|
||||
value="{{ default_search_queries }}"
|
||||
data-demo-disabled>
|
||||
<button id="clear-keywords" class="btn-ghost-neo"
|
||||
onclick="document.getElementById('keywords-input').value=''" type="button"
|
||||
data-demo-disabled>
|
||||
<i data-lucide="x" class="w-4 h-4"></i>
|
||||
</button>
|
||||
</div>
|
||||
<label class="label px-0">
|
||||
<span class="label-text-alt text-[#111111]/40 font-mono text-xs">Comma-separated topics. Leave empty for config default.</span>
|
||||
</label>
|
||||
</div>
|
||||
|
||||
<!-- Action Button -->
|
||||
<button id="create-btn" class="btn-primary-neo w-full h-16 text-lg"
|
||||
onclick="startPipeline()" disabled>
|
||||
<span id="btn-text">Initializing...</span>
|
||||
<span id="btn-spinner" class="loading loading-spinner loading-md hidden"></span>
|
||||
</button>
|
||||
|
||||
<!-- Progress Visualization -->
|
||||
<div id="progress-area" class="hidden space-y-4">
|
||||
<div class="flex justify-between items-end">
|
||||
<div class="space-y-1">
|
||||
<span class="text-xs uppercase tracking-widest text-[#111111]/40 font-mono font-bold">Current Stage</span>
|
||||
<div class="flex items-center gap-2">
|
||||
<div class="w-2 h-2 bg-[#DE6C56]"></div>
|
||||
<h3 id="stage-text" class="text-[#DE6C56] font-mono font-bold text-lg uppercase tracking-wider">Preparing...</h3>
|
||||
</div>
|
||||
</div>
|
||||
<span id="pct-text" class="text-2xl font-black text-[#111111]/20 font-mono">0%</span>
|
||||
</div>
|
||||
|
||||
<progress id="progress-bar" class="progress w-full h-3 bg-[#111111]/10 [&::-webkit-progress-value]:bg-[#DE6C56] [&::-moz-progress-bar]:bg-[#DE6C56]" value="0" max="100"></progress>
|
||||
|
||||
<div class="grid grid-cols-2 gap-4 pt-4">
|
||||
<div class="bg-white border border-[#111111] p-3 flex items-center gap-3">
|
||||
<i data-lucide="clock" class="w-4 h-4 text-[#111111]/40"></i>
|
||||
<div class="flex flex-col">
|
||||
<span class="text-[10px] uppercase text-[#111111]/40 font-mono font-bold">Elapsed</span>
|
||||
<span id="elapsed-time" class="text-sm font-mono text-[#111111]">00:00</span>
|
||||
</div>
|
||||
</div>
|
||||
<div class="bg-white border border-[#111111] p-3 flex items-center gap-3">
|
||||
<i data-lucide="layers" class="w-4 h-4 text-[#111111]/40"></i>
|
||||
<div class="flex flex-col">
|
||||
<span class="text-[10px] uppercase text-[#111111]/40 font-mono font-bold">Status</span>
|
||||
<span class="text-sm font-mono text-[#DE6C56]">Processing</span>
|
||||
</div>
|
||||
</div>
|
||||
</div>
|
||||
</div>
|
||||
|
||||
<!-- Success Message -->
|
||||
<div id="done-area" class="hidden">
|
||||
<div class="border border-[#4ADE80] bg-[#4ADE80]/10 p-6 flex flex-col items-start gap-4">
|
||||
<div class="flex items-center gap-3">
|
||||
<div class="bg-[#4ADE80] text-[#111111] p-1">
|
||||
<i data-lucide="check" class="w-4 h-4"></i>
|
||||
</div>
|
||||
<span class="font-display font-black uppercase tracking-tighter text-lg text-[#111111]">Generation Complete!</span>
|
||||
</div>
|
||||
<p id="done-msg" class="text-[#111111]/60 font-mono text-sm">Your video has been rendered and saved to the library.</p>
|
||||
<a href="{{ app_url('/') }}" class="btn-secondary-neo text-sm">View Video</a>
|
||||
</div>
|
||||
</div>
|
||||
|
||||
<!-- Error Message -->
|
||||
<div id="error-area" class="hidden">
|
||||
<div class="border border-[#DE6C56] bg-[#DE6C56]/5 p-6">
|
||||
<i data-lucide="alert-triangle" class="w-6 h-6 text-[#DE6C56]"></i>
|
||||
<div class="mt-2">
|
||||
<h3 class="font-display font-black uppercase tracking-tighter text-[#111111]">Pipeline Failed</h3>
|
||||
<div id="error-text" class="text-xs mt-2 font-mono bg-[#111111]/5 p-3 overflow-x-auto whitespace-pre-wrap text-[#111111]/70"></div>
|
||||
</div>
|
||||
</div>
|
||||
</div>
|
||||
|
||||
<!-- Log Output -->
|
||||
<div id="log-area" class="hidden space-y-3">
|
||||
<div class="flex items-center justify-between">
|
||||
<h4 class="text-xs uppercase tracking-widest text-[#111111]/40 font-mono font-bold">Execution Logs</h4>
|
||||
<span class="font-mono text-[10px] uppercase border border-[#111111] px-2 py-0.5 text-[#111111]/50">Real-time</span>
|
||||
</div>
|
||||
<div id="log-list" class="bg-white border border-[#111111] p-4 font-mono text-[11px] leading-relaxed text-[#111111]/60 h-48 overflow-y-auto">
|
||||
</div>
|
||||
</div>
|
||||
</div>
|
||||
</div>
|
||||
</div>
|
||||
</div>
|
||||
|
||||
<!-- RIGHT PANEL: Pipeline Activity Visualization -->
|
||||
<div class="lg:col-span-3">
|
||||
<div class="card-neo">
|
||||
<div class="p-6">
|
||||
<div class="flex items-center justify-between mb-5">
|
||||
<div class="flex items-center gap-2">
|
||||
<i data-lucide="activity" class="w-5 h-5 text-[#DE6C56]"></i>
|
||||
<h3 class="font-display font-black uppercase tracking-tighter text-lg text-[#111111]">Pipeline Activity</h3>
|
||||
</div>
|
||||
<span class="font-mono text-[10px] uppercase border border-[#4ADE80] text-[#4ADE80] px-2 py-0.5">Live</span>
|
||||
</div>
|
||||
|
||||
<!-- Stage Diagram -->
|
||||
<div id="stage-diagram" class="mb-6">
|
||||
</div>
|
||||
|
||||
<!-- Scraper Event Feed -->
|
||||
<div id="scraper-feed" class="space-y-2 max-h-[500px] overflow-y-auto pr-1">
|
||||
</div>
|
||||
|
||||
<!-- Empty state -->
|
||||
<div id="feed-empty" class="text-center py-16 text-[#111111]/30">
|
||||
<i data-lucide="eye-off" class="w-12 h-12 mx-auto mb-3"></i>
|
||||
<p class="text-sm font-mono uppercase tracking-wider">Scraper activity will appear here</p>
|
||||
<p class="text-xs mt-1 font-mono">Start a pipeline to see real-time scraping visualization</p>
|
||||
</div>
|
||||
</div>
|
||||
</div>
|
||||
</div>
|
||||
|
||||
</div>
|
||||
</div>
|
||||
</div>
|
||||
|
||||
<script>
|
||||
let pollTimer = null;
|
||||
let startTime = null;
|
||||
let elapsedTimer = null;
|
||||
|
||||
const stageWeights = {
|
||||
'configuring': 5,
|
||||
'discovering': 15,
|
||||
'scraping': 20,
|
||||
'fetching': 25,
|
||||
'saving': 35,
|
||||
'tts': 45,
|
||||
'screenshots': 60,
|
||||
'background': 70,
|
||||
'chopping': 75,
|
||||
'creating': 80,
|
||||
'rendering': 90,
|
||||
'done': 100,
|
||||
'error': 0
|
||||
};
|
||||
|
||||
const pipelineStages = [
|
||||
{ id: 'configuring', label: 'Configuring', icon: 'settings' },
|
||||
{ id: 'discovering', label: 'Discovering', icon: 'search' },
|
||||
{ id: 'scraping', label: 'Scraping', icon: 'loader-2' },
|
||||
{ id: 'fetching', label: 'Fetching', icon: 'download' },
|
||||
{ id: 'tts', label: 'TTS', icon: 'volume-2' },
|
||||
{ id: 'screenshots', label: 'Screenshots', icon: 'camera' },
|
||||
{ id: 'background', label: 'Background', icon: 'image' },
|
||||
{ id: 'chopping', label: 'Chopping', icon: 'scissors' },
|
||||
{ id: 'creating', label: 'Creating', icon: 'film' },
|
||||
{ id: 'rendering', label: 'Rendering', icon: 'sparkles' },
|
||||
{ id: 'done', label: 'Complete', icon: 'check-circle' },
|
||||
];
|
||||
|
||||
// --- helpers ---
|
||||
function esc(s) {
|
||||
if (!s) return '';
|
||||
const div = document.createElement('div');
|
||||
div.textContent = s;
|
||||
return div.innerHTML;
|
||||
}
|
||||
|
||||
function fmtMetric(n) {
|
||||
if (!n && n !== 0) return '0';
|
||||
n = Number(n);
|
||||
if (n >= 1000000) return (n / 1000000).toFixed(1) + 'M';
|
||||
if (n >= 1000) return (n / 1000).toFixed(1) + 'K';
|
||||
return n.toLocaleString();
|
||||
}
|
||||
|
||||
function fmtTime(ts) {
|
||||
if (!ts) return '';
|
||||
const seconds = Math.floor((Date.now() / 1000) - ts);
|
||||
if (seconds < 5) return 'just now';
|
||||
if (seconds < 60) return seconds + 's ago';
|
||||
const mins = Math.floor(seconds / 60);
|
||||
return mins + 'm ago';
|
||||
}
|
||||
|
||||
function updateElapsedTime() {
|
||||
if (!startTime) return;
|
||||
const now = new Date();
|
||||
const diff = Math.floor((now - startTime) / 1000);
|
||||
const mins = Math.floor(diff / 60).toString().padStart(2, '0');
|
||||
const secs = (diff % 60).toString().padStart(2, '0');
|
||||
document.getElementById('elapsed-time').textContent = mins + ':' + secs;
|
||||
}
|
||||
|
||||
function stageProgress(stage) {
|
||||
let pct = 0;
|
||||
const s = (stage || '').toLowerCase();
|
||||
for (let [key, val] of Object.entries(stageWeights)) {
|
||||
if (s.includes(key)) { pct = val; }
|
||||
}
|
||||
return pct;
|
||||
}
|
||||
|
||||
// --- stage diagram ---
|
||||
function renderStageDiagram(currentStage) {
|
||||
const container = document.getElementById('stage-diagram');
|
||||
const s = (currentStage || '').toLowerCase();
|
||||
|
||||
let activeIdx = -1;
|
||||
for (let i = 0; i < pipelineStages.length; i++) {
|
||||
if (s.includes(pipelineStages[i].id)) {
|
||||
activeIdx = i;
|
||||
}
|
||||
}
|
||||
|
||||
const start = Math.max(0, activeIdx - 4);
|
||||
const end = Math.min(pipelineStages.length - 1, Math.max(activeIdx, start + 7));
|
||||
const visible = pipelineStages.slice(start, end + 1);
|
||||
|
||||
let html = '<div class="flex items-center gap-1 overflow-x-auto py-2">';
|
||||
visible.forEach((st, idx) => {
|
||||
const globalIdx = start + idx;
|
||||
const isDone = globalIdx < activeIdx;
|
||||
const isActive = globalIdx === activeIdx;
|
||||
const isPending = globalIdx > activeIdx;
|
||||
|
||||
let cls = 'flex items-center gap-1.5 px-2.5 py-1.5 text-xs font-mono font-medium whitespace-nowrap shrink-0 transition-all duration-300 border ';
|
||||
if (isActive) {
|
||||
cls += 'bg-[#DE6C56]/10 border-[#DE6C56] text-[#DE6C56]';
|
||||
} else if (isDone) {
|
||||
cls += 'bg-[#4ADE80]/10 border-[#4ADE80] text-[#111111]';
|
||||
} else {
|
||||
cls += 'bg-white border-[#111111]/20 text-[#111111]/40';
|
||||
}
|
||||
|
||||
html += '<div class="' + cls + '">';
|
||||
if (isDone) {
|
||||
html += '<i data-lucide="check" class="w-3 h-3 text-[#4ADE80]"></i>';
|
||||
} else if (isActive) {
|
||||
html += '<i data-lucide="loader-2" class="w-3 h-3 text-[#DE6C56] animate-spin"></i>';
|
||||
} else {
|
||||
html += '<i data-lucide="' + st.icon + '" class="w-3 h-3"></i>';
|
||||
}
|
||||
html += '<span class="uppercase tracking-wider">' + st.label + '</span></div>';
|
||||
|
||||
if (idx < visible.length - 1) {
|
||||
html += '<div class="text-[#111111]/30 shrink-0 mx-0.5">///</div>';
|
||||
}
|
||||
});
|
||||
html += '</div>';
|
||||
|
||||
container.innerHTML = html;
|
||||
lucide.createIcons();
|
||||
}
|
||||
|
||||
// --- scraper event feed ---
|
||||
function renderScraperEvent(event) {
|
||||
const { type, data, ts } = event;
|
||||
const d = data || {};
|
||||
|
||||
const templates = {
|
||||
'post_discovered': () =>
|
||||
'<div class="bg-white border border-[#111111] p-3 hover:border-[#DE6C56] transition-colors">' +
|
||||
'<div class="flex items-start gap-3">' +
|
||||
'<div class="bg-[#DE6C56]/10 p-1.5 border border-[#DE6C56] shrink-0">' +
|
||||
'<i data-lucide="message-circle" class="w-3.5 h-3.5 text-[#DE6C56]"></i>' +
|
||||
'</div>' +
|
||||
'<div class="flex-1 min-w-0">' +
|
||||
'<div class="flex items-center gap-2 mb-1">' +
|
||||
'<span class="text-xs font-mono font-bold text-[#111111] truncate uppercase">' + esc(d.username || 'unknown') + '</span>' +
|
||||
'<span class="text-[10px] text-[#111111]/40 font-mono">' + fmtTime(ts) + '</span>' +
|
||||
'</div>' +
|
||||
'<p class="text-[11px] text-[#111111]/60 font-mono leading-relaxed line-clamp-2">' + esc(d.body || '') + '</p>' +
|
||||
'<div class="flex items-center gap-3 mt-1.5 text-[10px] text-[#111111]/40 font-mono">' +
|
||||
'<span>♥ ' + fmtMetric(d.likes) + '</span>' +
|
||||
'<span>💬 ' + fmtMetric(d.replies) + '</span>' +
|
||||
(d.reposts ? '<span>🔃 ' + fmtMetric(d.reposts) + '</span>' : '') +
|
||||
'</div>' +
|
||||
'</div>' +
|
||||
'</div>' +
|
||||
'</div>',
|
||||
|
||||
'feed_scroll': () =>
|
||||
'<div class="flex items-center gap-2 py-1.5 px-1 font-mono">' +
|
||||
'<div class="bg-white border border-[#111111] p-1">' +
|
||||
'<i data-lucide="mouse-pointer-2" class="w-3 h-3 text-[#DE6C56]"></i>' +
|
||||
'</div>' +
|
||||
'<span class="text-[11px] text-[#111111]/50">' +
|
||||
'Scrolled <strong class="text-[#111111]">' + (d.scroll || '?') + '/' + (d.max_scrolls || '?') + '</strong>' +
|
||||
' — <strong class="text-[#111111]">' + (d.new_posts || 0) + '</strong> new,' +
|
||||
' <strong class="text-[#111111]">' + (d.total_posts || 0) + '</strong> total posts' +
|
||||
'</span>' +
|
||||
'</div>',
|
||||
|
||||
'search_query': () =>
|
||||
'<div class="flex items-center gap-2 py-1.5 px-1 font-mono">' +
|
||||
'<div class="bg-white border border-[#111111] p-1">' +
|
||||
'<i data-lucide="search" class="w-3 h-3 text-[#111111]"></i>' +
|
||||
'</div>' +
|
||||
'<span class="text-[11px] text-[#111111]/50">' +
|
||||
'Searched "<strong class="text-[#111111]">' + esc(d.query || '') + '</strong>"' +
|
||||
' — <strong class="text-[#111111]">' + (d.posts_found || 0) + '</strong> posts found' +
|
||||
'</span>' +
|
||||
'</div>',
|
||||
|
||||
'filter_results': () =>
|
||||
'<div class="flex items-center gap-2 py-1.5 px-1 font-mono">' +
|
||||
'<div class="bg-white border border-[#111111] p-1">' +
|
||||
'<i data-lucide="filter" class="w-3 h-3 text-[#111111]"></i>' +
|
||||
'</div>' +
|
||||
'<span class="text-[11px] text-[#111111]/50">' +
|
||||
'Filtered <strong class="text-[#111111]">' + (d.before || 0) + '</strong> posts →' +
|
||||
' <strong class="text-[#111111]">' + (d.after || 0) + '</strong> candidates' +
|
||||
(d.min_engagement ? ' (min ' + fmtMetric(d.min_engagement) + ' engagement)' : '') +
|
||||
'</span>' +
|
||||
'</div>',
|
||||
|
||||
'visiting_post': () =>
|
||||
'<div class="flex items-center gap-2 py-1.5 px-1 font-mono">' +
|
||||
'<div class="bg-white border border-[#111111] p-1">' +
|
||||
'<i data-lucide="external-link" class="w-3 h-3 text-[#DE6C56]"></i>' +
|
||||
'</div>' +
|
||||
'<span class="text-[11px] text-[#111111]/50">' +
|
||||
'Examining candidate #' + (d.attempt || '?') + ': "' + esc((d.body || '').substring(0, 40)) + '..."' +
|
||||
' <span class="text-[#111111]/40 ml-1">♥' + fmtMetric(d.likes) + '</span>' +
|
||||
'</span>' +
|
||||
'</div>',
|
||||
|
||||
'replies_found': () =>
|
||||
'<div class="flex items-center gap-2 py-1.5 px-1 font-mono">' +
|
||||
'<div class="bg-white border border-[#111111] p-1">' +
|
||||
'<i data-lucide="message-square" class="w-3 h-3 text-[#4ADE80]"></i>' +
|
||||
'</div>' +
|
||||
'<span class="text-[11px] text-[#111111]/50">' +
|
||||
d.count + ' replies found' + (d.min_required ? ' (need ' + d.min_required + ')' : '') +
|
||||
'</span>' +
|
||||
'</div>',
|
||||
|
||||
'post_selected': () =>
|
||||
'<div class="bg-[#4ADE80]/10 border border-[#4ADE80] p-3">' +
|
||||
'<div class="flex items-start gap-3">' +
|
||||
'<div class="bg-[#4ADE80] p-1.5 shrink-0">' +
|
||||
'<i data-lucide="check-circle" class="w-3.5 h-3.5 text-[#111111]"></i>' +
|
||||
'</div>' +
|
||||
'<div>' +
|
||||
'<span class="text-xs font-mono font-bold text-[#111111] uppercase">Post Selected!</span>' +
|
||||
'<p class="text-[11px] text-[#111111]/60 font-mono mt-0.5">' + esc(d.title || '') + '</p>' +
|
||||
'<div class="flex gap-3 mt-1 text-[10px] text-[#111111]/40 font-mono">' +
|
||||
'<span>♥ ' + fmtMetric(d.likes) + '</span>' +
|
||||
'<span>💬 ' + (d.replies_count || 0) + ' replies</span>' +
|
||||
'</div>' +
|
||||
'</div>' +
|
||||
'</div>' +
|
||||
'</div>',
|
||||
|
||||
'login': () =>
|
||||
'<div class="flex items-center gap-2 py-1.5 px-1 font-mono">' +
|
||||
'<div class="bg-white border border-[#111111] p-1">' +
|
||||
'<i data-lucide="log-in" class="w-3 h-3 text-[#111111]"></i>' +
|
||||
'</div>' +
|
||||
'<span class="text-[11px] text-[#111111]/50">' + esc(d.message || '') + '</span>' +
|
||||
'</div>',
|
||||
|
||||
'browser_launch': () =>
|
||||
'<div class="flex items-center gap-2 py-1.5 px-1 font-mono">' +
|
||||
'<div class="bg-white border border-[#111111] p-1">' +
|
||||
'<i data-lucide="globe" class="w-3 h-3 text-[#DE6C56]"></i>' +
|
||||
'</div>' +
|
||||
'<span class="text-[11px] text-[#111111]/50">' + esc(d.message || '') + '</span>' +
|
||||
'</div>',
|
||||
};
|
||||
|
||||
const fn = templates[type];
|
||||
return fn ? fn() : (
|
||||
'<div class="flex items-center gap-2 py-1.5 px-1 font-mono">' +
|
||||
'<span class="text-[11px] text-[#111111]/50">' + esc(d.message || type) + '</span>' +
|
||||
'</div>'
|
||||
);
|
||||
}
|
||||
|
||||
function renderScraperFeed(events) {
|
||||
const container = document.getElementById('scraper-feed');
|
||||
const empty = document.getElementById('feed-empty');
|
||||
|
||||
if (!events || events.length === 0) {
|
||||
container.innerHTML = '';
|
||||
empty.classList.remove('hidden');
|
||||
return;
|
||||
}
|
||||
|
||||
empty.classList.add('hidden');
|
||||
|
||||
const recent = events.slice(-50);
|
||||
container.innerHTML = recent.map(function(e) { return renderScraperEvent(e); }).join('');
|
||||
container.scrollTop = container.scrollHeight;
|
||||
|
||||
lucide.createIcons();
|
||||
}
|
||||
|
||||
// --- pipeline lifecycle ---
|
||||
async function startPipeline() {
|
||||
const btn = document.getElementById('create-btn');
|
||||
const btnText = document.getElementById('btn-text');
|
||||
const spinner = document.getElementById('btn-spinner');
|
||||
|
||||
btn.disabled = true;
|
||||
spinner.classList.remove('hidden');
|
||||
btnText.textContent = 'Initializing...';
|
||||
|
||||
document.getElementById('progress-area').classList.remove('hidden');
|
||||
document.getElementById('log-area').classList.remove('hidden');
|
||||
document.getElementById('done-area').classList.add('hidden');
|
||||
document.getElementById('error-area').classList.add('hidden');
|
||||
|
||||
// Show empty feed state
|
||||
document.getElementById('scraper-feed').innerHTML = '';
|
||||
document.getElementById('feed-empty').classList.remove('hidden');
|
||||
document.getElementById('stage-diagram').innerHTML = '';
|
||||
|
||||
// Reset progress
|
||||
document.getElementById('progress-bar').classList.remove('progress-success', 'progress-error');
|
||||
|
||||
const keywords = document.getElementById('keywords-input').value.trim();
|
||||
|
||||
try {
|
||||
const r = await fetch(window.appPath('/create'), {
|
||||
method: 'POST',
|
||||
headers: { 'Content-Type': 'application/json' },
|
||||
body: JSON.stringify({ search_queries: keywords || null }),
|
||||
});
|
||||
const data = await r.json();
|
||||
|
||||
if (data.status === 'started' || data.status === 'already_running') {
|
||||
btnText.textContent = 'Processing...';
|
||||
startTime = new Date();
|
||||
elapsedTimer = setInterval(updateElapsedTime, 1000);
|
||||
pollTimer = setInterval(pollStatus, 2000);
|
||||
}
|
||||
} catch (err) {
|
||||
console.error("Failed to start pipeline:", err);
|
||||
btn.disabled = false;
|
||||
spinner.classList.add('hidden');
|
||||
btnText.textContent = 'Retry';
|
||||
}
|
||||
}
|
||||
|
||||
async function pollStatus() {
|
||||
try {
|
||||
const r = await fetch(window.appPath('/create/status'));
|
||||
const state = await r.json();
|
||||
|
||||
const stageText = document.getElementById('stage-text');
|
||||
const progressBar = document.getElementById('progress-bar');
|
||||
const pctText = document.getElementById('pct-text');
|
||||
const logList = document.getElementById('log-list');
|
||||
|
||||
stageText.textContent = state.stage || 'Running...';
|
||||
const pct = stageProgress(state.stage || '');
|
||||
progressBar.value = pct;
|
||||
pctText.textContent = pct + '%';
|
||||
|
||||
if (state.log && state.log.length > 0) {
|
||||
logList.innerHTML = state.log.map(function(l) {
|
||||
return '<div class="py-0.5 border-b border-[#111111]/10 last:border-0">' + esc(l) + '</div>';
|
||||
}).join('');
|
||||
logList.scrollTop = logList.scrollHeight;
|
||||
}
|
||||
|
||||
// Render visualization panels
|
||||
renderStageDiagram(state.stage);
|
||||
if (state.scraper_events) {
|
||||
renderScraperFeed(state.scraper_events);
|
||||
}
|
||||
|
||||
if (!state.running) {
|
||||
clearInterval(pollTimer);
|
||||
clearInterval(elapsedTimer);
|
||||
pollTimer = null;
|
||||
|
||||
document.getElementById('btn-spinner').classList.add('hidden');
|
||||
|
||||
if (state.stage === 'done' || state.result) {
|
||||
progressBar.value = 100;
|
||||
progressBar.classList.add('progress-success');
|
||||
document.getElementById('btn-text').textContent = 'Create New';
|
||||
document.getElementById('done-area').classList.remove('hidden');
|
||||
if (state.result) {
|
||||
document.getElementById('done-msg').textContent = state.result.message;
|
||||
}
|
||||
} else if (state.error) {
|
||||
progressBar.classList.add('progress-error');
|
||||
document.getElementById('btn-text').textContent = 'Retry';
|
||||
document.getElementById('error-area').classList.remove('hidden');
|
||||
document.getElementById('error-text').textContent = state.error;
|
||||
}
|
||||
document.getElementById('create-btn').disabled = false;
|
||||
}
|
||||
} catch (err) {
|
||||
console.error("Status poll failed:", err);
|
||||
}
|
||||
}
|
||||
|
||||
// --- init ---
|
||||
window.addEventListener('load', async function() {
|
||||
lucide.createIcons();
|
||||
try {
|
||||
const r = await fetch(window.appPath('/create/status'));
|
||||
const state = await r.json();
|
||||
const btn = document.getElementById('create-btn');
|
||||
|
||||
if (window.PUBLIC_DEMO_MODE) {
|
||||
btn.disabled = true;
|
||||
document.getElementById('btn-text').textContent = 'Public Demo: Disabled';
|
||||
} else if (state.running) {
|
||||
document.getElementById('progress-area').classList.remove('hidden');
|
||||
document.getElementById('log-area').classList.remove('hidden');
|
||||
btn.disabled = true;
|
||||
document.getElementById('btn-spinner').classList.remove('hidden');
|
||||
document.getElementById('btn-text').textContent = 'Running...';
|
||||
startTime = new Date();
|
||||
elapsedTimer = setInterval(updateElapsedTime, 1000);
|
||||
pollTimer = setInterval(pollStatus, 2000);
|
||||
renderStageDiagram(state.stage);
|
||||
if (state.scraper_events) {
|
||||
renderScraperFeed(state.scraper_events);
|
||||
}
|
||||
} else {
|
||||
btn.disabled = false;
|
||||
document.getElementById('btn-text').textContent = 'Start Generation';
|
||||
}
|
||||
} catch (err) {
|
||||
console.error("Initial status check failed:", err);
|
||||
document.getElementById('create-btn').disabled = window.PUBLIC_DEMO_MODE;
|
||||
document.getElementById('btn-text').textContent = window.PUBLIC_DEMO_MODE ? 'Public Demo: Disabled' : 'Start Generation';
|
||||
}
|
||||
});
|
||||
</script>
|
||||
{% endblock %}
|
||||
@ -1,155 +1,395 @@
|
||||
<!DOCTYPE html>
|
||||
<html lang="en">
|
||||
|
||||
<head>
|
||||
<meta charset="utf-8">
|
||||
<meta name="viewport" content="width=device-width, initial-scale=1">
|
||||
<meta http-equiv="cache-control" content="no-cache" />
|
||||
<title>RedditVideoMakerBot</title>
|
||||
<link rel="stylesheet" href="https://cdn.jsdelivr.net/npm/bootstrap@4.1.3/dist/css/bootstrap.min.css"
|
||||
integrity="sha384-MCw98/SFnGE8fJT3GXwEOngsV7Zt27NXFoaoApmYm81iuXoPkFOJwJ8ERdknLPMO" crossorigin="anonymous">
|
||||
<link href="https://getbootstrap.com/docs/5.2/dist/css/bootstrap.min.css" rel="stylesheet">
|
||||
<link rel="stylesheet" href="https://cdn.jsdelivr.net/npm/bootstrap-icons@1.9.1/font/bootstrap-icons.css">
|
||||
<title>VideoMakerBot</title>
|
||||
|
||||
<!-- Tailwind config — must load BEFORE the CDN script -->
|
||||
<script>
|
||||
tailwind = window.tailwind || {};
|
||||
tailwind.config = {
|
||||
theme: {
|
||||
extend: {
|
||||
colors: {
|
||||
cream: '#F2EFEB',
|
||||
jet: '#111111',
|
||||
terracotta: '#DE6C56',
|
||||
'terracotta-dark': '#C85A46',
|
||||
terminal: '#4ADE80',
|
||||
},
|
||||
fontFamily: {
|
||||
display: ['"Clash Display"', 'system-ui', 'sans-serif'],
|
||||
mono: ['"Space Mono"', 'monospace'],
|
||||
},
|
||||
}
|
||||
}
|
||||
};
|
||||
</script>
|
||||
<script src="https://cdn.tailwindcss.com"></script>
|
||||
|
||||
<!-- DaisyUI (component library) -->
|
||||
<link href="https://cdn.jsdelivr.net/npm/daisyui@4.4.19/dist/full.min.css" rel="stylesheet" type="text/css" />
|
||||
|
||||
<!-- Fonts: Clash Display (display/headings via Fontshare) + Space Mono (body/UI via Google Fonts) -->
|
||||
<link rel="preconnect" href="https://api.fontshare.com">
|
||||
<link rel="preconnect" href="https://fonts.googleapis.com">
|
||||
<link rel="preconnect" href="https://fonts.gstatic.com" crossorigin>
|
||||
<link href="https://api.fontshare.com/v2/css?f[]=clash-display@400,500,600,700&display=swap" rel="stylesheet">
|
||||
<link href="https://fonts.googleapis.com/css2?family=Space+Mono:wght@400;700&display=swap" rel="stylesheet">
|
||||
|
||||
<!-- Lucide icons -->
|
||||
<script src="https://unpkg.com/lucide@latest"></script>
|
||||
<script>
|
||||
window.PUBLIC_BASE_PATH = {{ public_base_path|tojson }};
|
||||
window.PUBLIC_DEMO_MODE = {{ public_demo_mode|tojson }};
|
||||
window.appPath = function(path) {
|
||||
var normalized = path.charAt(0) === '/' ? path : '/' + path;
|
||||
return window.PUBLIC_BASE_PATH + normalized;
|
||||
};
|
||||
document.addEventListener('DOMContentLoaded', function() {
|
||||
if (!window.PUBLIC_DEMO_MODE) return;
|
||||
document.querySelectorAll('[data-demo-disabled]').forEach(function(el) {
|
||||
el.disabled = true;
|
||||
});
|
||||
});
|
||||
</script>
|
||||
|
||||
<style>
|
||||
.bd-placeholder-img {
|
||||
font-size: 1.125rem;
|
||||
text-anchor: middle;
|
||||
-webkit-user-select: none;
|
||||
-moz-user-select: none;
|
||||
user-select: none;
|
||||
/* ── Neo-Brutalist Global Overrides ───────────────────────────── */
|
||||
|
||||
/* DaisyUI border-radius kill switch */
|
||||
:root {
|
||||
--rounded-box: 0rem;
|
||||
--rounded-btn: 0rem;
|
||||
--rounded-badge: 0rem;
|
||||
--tab-radius: 0rem;
|
||||
--tooltip-radius: 0rem;
|
||||
--alert-radius: 0rem;
|
||||
--input-radius: 0rem;
|
||||
}
|
||||
|
||||
.feedback-invalid {
|
||||
color: #dc3545;
|
||||
/* Global reset */
|
||||
body {
|
||||
font-family: 'Space Mono', monospace;
|
||||
background-color: #F2EFEB;
|
||||
color: #111111;
|
||||
}
|
||||
|
||||
@media (min-width: 768px) {
|
||||
.bd-placeholder-img-lg {
|
||||
font-size: 3.5rem;
|
||||
}
|
||||
/* Kill all box-shadows */
|
||||
*,
|
||||
*::before,
|
||||
*::after {
|
||||
box-shadow: none !important;
|
||||
}
|
||||
|
||||
.bi {
|
||||
vertical-align: -.125em;
|
||||
fill: currentColor;
|
||||
/* ── Typography ────────────────────────────────────────────────── */
|
||||
|
||||
h1, h2, h3 {
|
||||
font-family: 'Clash Display', system-ui, sans-serif;
|
||||
font-weight: 900;
|
||||
text-transform: uppercase;
|
||||
letter-spacing: -0.025em;
|
||||
}
|
||||
|
||||
.nav {
|
||||
display: flex;
|
||||
flex-wrap: nowrap;
|
||||
padding-bottom: 1rem;
|
||||
margin-top: -1px;
|
||||
overflow-x: auto;
|
||||
text-align: center;
|
||||
white-space: nowrap;
|
||||
-webkit-overflow-scrolling: touch;
|
||||
/* All UI text: monospace */
|
||||
button, a, input, select, textarea, label, .btn, .badge, nav, footer {
|
||||
font-family: 'Space Mono', monospace !important;
|
||||
}
|
||||
|
||||
/* ── DaisyUI Component Overrides ───────────────────────────────── */
|
||||
|
||||
/* Buttons — kill radius, uppercase */
|
||||
.btn {
|
||||
border-radius: 0 !important;
|
||||
text-transform: uppercase;
|
||||
letter-spacing: 0.025em;
|
||||
font-weight: 500;
|
||||
}
|
||||
|
||||
/* Cards */
|
||||
.card {
|
||||
border-radius: 0 !important;
|
||||
}
|
||||
|
||||
/* Inputs / Selects / Textareas */
|
||||
.input, .select, .textarea {
|
||||
border-radius: 0 !important;
|
||||
border: 1px solid #111111;
|
||||
}
|
||||
|
||||
/* Badges */
|
||||
.badge {
|
||||
border-radius: 0 !important;
|
||||
}
|
||||
|
||||
/* Modals */
|
||||
.modal-box {
|
||||
border-radius: 0 !important;
|
||||
}
|
||||
|
||||
/* Menu items */
|
||||
.menu li > * {
|
||||
border-radius: 0 !important;
|
||||
}
|
||||
|
||||
/* Toggle / Checkbox — kill radius */
|
||||
.toggle {
|
||||
border-radius: 0 !important;
|
||||
}
|
||||
|
||||
/* Alerts */
|
||||
.alert {
|
||||
border-radius: 0 !important;
|
||||
}
|
||||
|
||||
/* Dividers — use /// text pattern where plain dividers appear */
|
||||
.divider:not(.divider-text)::after,
|
||||
.divider:not(.divider-text)::before {
|
||||
/* We'll use .divider-text overrides in templates */
|
||||
}
|
||||
|
||||
/* Progress bar */
|
||||
.progress {
|
||||
border-radius: 0 !important;
|
||||
}
|
||||
|
||||
/* Join groups */
|
||||
.join > * {
|
||||
border-radius: 0 !important;
|
||||
}
|
||||
|
||||
/* ── Custom Scrollbar ──────────────────────────────────────────── */
|
||||
::-webkit-scrollbar { width: 8px; }
|
||||
::-webkit-scrollbar-track { background: #F2EFEB; }
|
||||
::-webkit-scrollbar-thumb { background: #111111; }
|
||||
::-webkit-scrollbar-thumb:hover { background: #333333; }
|
||||
|
||||
/* Dotted path inputs — suppress default browser validation UI */
|
||||
input:invalid { box-shadow: none; }
|
||||
|
||||
/* ── Navigation ────────────────────────────────────────────────── */
|
||||
.nav-link {
|
||||
font-family: 'Space Mono', monospace;
|
||||
font-size: 0.8125rem;
|
||||
color: #111111;
|
||||
text-transform: uppercase;
|
||||
letter-spacing: 0.05em;
|
||||
padding: 0.375rem 0.75rem;
|
||||
transition: background-color 150ms;
|
||||
}
|
||||
.nav-link:hover {
|
||||
background-color: #111111;
|
||||
color: #F2EFEB;
|
||||
}
|
||||
|
||||
#tooltip {
|
||||
background-color: #333;
|
||||
color: white;
|
||||
padding: 5px 10px;
|
||||
border-radius: 4px;
|
||||
font-size: 13px;
|
||||
/* ── Buttons ───────────────────────────────────────────────────── */
|
||||
.btn-primary-neo {
|
||||
background-color: #DE6C56;
|
||||
color: #F2EFEB;
|
||||
border: 1px solid #DE6C56;
|
||||
display: inline-flex;
|
||||
align-items: center;
|
||||
justify-content: center;
|
||||
font-family: 'Space Mono', monospace;
|
||||
text-transform: uppercase;
|
||||
font-weight: 500;
|
||||
letter-spacing: 0.025em;
|
||||
padding: 0.5rem 1.25rem;
|
||||
cursor: pointer;
|
||||
transition: all 150ms;
|
||||
border-radius: 0;
|
||||
}
|
||||
.btn-primary-neo:hover {
|
||||
background-color: #111111;
|
||||
border-color: #111111;
|
||||
color: #F2EFEB;
|
||||
}
|
||||
|
||||
.tooltip-inner {
|
||||
max-width: 500px !important;
|
||||
.btn-secondary-neo {
|
||||
background-color: transparent;
|
||||
color: #111111;
|
||||
border: 2px solid #111111;
|
||||
display: inline-flex;
|
||||
align-items: center;
|
||||
justify-content: center;
|
||||
font-family: 'Space Mono', monospace;
|
||||
text-transform: uppercase;
|
||||
font-weight: 500;
|
||||
letter-spacing: 0.025em;
|
||||
padding: 0.5rem 1.25rem;
|
||||
cursor: pointer;
|
||||
transition: all 150ms;
|
||||
border-radius: 0;
|
||||
}
|
||||
.btn-secondary-neo:hover {
|
||||
background-color: #111111;
|
||||
color: #F2EFEB;
|
||||
}
|
||||
#hard-reload {
|
||||
|
||||
.btn-ghost-neo {
|
||||
background: transparent;
|
||||
color: #111111;
|
||||
border: 1px solid #111111;
|
||||
display: inline-flex;
|
||||
align-items: center;
|
||||
justify-content: center;
|
||||
font-family: 'Space Mono', monospace;
|
||||
text-transform: uppercase;
|
||||
font-size: 0.75rem;
|
||||
padding: 0.375rem 0.75rem;
|
||||
cursor: pointer;
|
||||
color: darkblue;
|
||||
border-radius: 0;
|
||||
transition: all 150ms;
|
||||
}
|
||||
.btn-ghost-neo:hover {
|
||||
background-color: #111111;
|
||||
color: #F2EFEB;
|
||||
}
|
||||
|
||||
.btn-danger-neo {
|
||||
background-color: transparent;
|
||||
color: #DE6C56;
|
||||
border: 1px solid #DE6C56;
|
||||
display: inline-flex;
|
||||
align-items: center;
|
||||
justify-content: center;
|
||||
font-family: 'Space Mono', monospace;
|
||||
text-transform: uppercase;
|
||||
font-weight: 500;
|
||||
letter-spacing: 0.025em;
|
||||
padding: 0.5rem 1.25rem;
|
||||
cursor: pointer;
|
||||
transition: all 150ms;
|
||||
border-radius: 0;
|
||||
}
|
||||
.btn-danger-neo:hover {
|
||||
background-color: #DE6C56;
|
||||
color: #F2EFEB;
|
||||
}
|
||||
.btn-danger-neo:disabled {
|
||||
opacity: 0.45;
|
||||
cursor: not-allowed;
|
||||
background-color: transparent;
|
||||
color: #DE6C56;
|
||||
}
|
||||
|
||||
/* ── Cards ─────────────────────────────────────────────────────── */
|
||||
.card-neo {
|
||||
background-color: #FFFFFF;
|
||||
border: 1px solid #111111;
|
||||
border-radius: 0;
|
||||
}
|
||||
|
||||
/* ── Inputs ────────────────────────────────────────────────────── */
|
||||
.input-neo {
|
||||
background-color: #FFFFFF;
|
||||
border: 1px solid #111111;
|
||||
color: #111111;
|
||||
font-family: 'Space Mono', monospace;
|
||||
border-radius: 0;
|
||||
padding: 0.5rem 0.75rem;
|
||||
font-size: 0.8125rem;
|
||||
}
|
||||
#hard-reload:hover {
|
||||
color: blue;
|
||||
.input-neo::placeholder { color: #999999; }
|
||||
.input-neo:focus {
|
||||
outline: none;
|
||||
border-width: 2px;
|
||||
}
|
||||
|
||||
/* ── Footer Ticker ─────────────────────────────────────────────── */
|
||||
.footer-ticker {
|
||||
background-color: #111111;
|
||||
color: #F2EFEB;
|
||||
font-family: 'Space Mono', monospace;
|
||||
font-size: 0.6875rem;
|
||||
letter-spacing: 0.05em;
|
||||
overflow: hidden;
|
||||
white-space: nowrap;
|
||||
padding: 0.625rem 0;
|
||||
}
|
||||
.footer-ticker a {
|
||||
color: #4ADE80;
|
||||
text-decoration: none;
|
||||
}
|
||||
.footer-ticker a:hover {
|
||||
color: #F2EFEB;
|
||||
text-decoration: underline;
|
||||
}
|
||||
</style>
|
||||
</head>
|
||||
|
||||
<script src="https://code.jquery.com/jquery-3.1.1.js" integrity="sha256-16cdPddA6VdVInumRGo6IbivbERE8p7CQR3HzTBuELA="
|
||||
crossorigin="anonymous"></script>
|
||||
<script src="https://cdn.jsdelivr.net/npm/popper.js@1.14.3/dist/umd/popper.min.js"
|
||||
integrity="sha384-ZMP7rVo3mIykV+2+9J3UJ46jBk0WLaUAdn689aCwoqbBJiSnjAK/l8WvCWPIPm49"
|
||||
crossorigin="anonymous"></script>
|
||||
<script src="https://cdn.jsdelivr.net/npm/bootstrap@4.1.3/dist/js/bootstrap.min.js"
|
||||
integrity="sha384-ChfqqxuZUCnJSK3+MXmPNIyE6ZbWh2IMqE241rYiqJxyMiZ6OW/JmZQ5stwEULTy"
|
||||
crossorigin="anonymous"></script>
|
||||
<script src="https://cdnjs.cloudflare.com/ajax/libs/clipboard.js/2.0.10/clipboard.min.js"></script>
|
||||
<script src="https://unpkg.com/isotope-layout@3/dist/isotope.pkgd.js"></script>
|
||||
|
||||
<body>
|
||||
<header>
|
||||
{% if get_flashed_messages() %}
|
||||
{% for category, message in get_flashed_messages(with_categories=true) %}
|
||||
|
||||
{% if category == "error" %}
|
||||
<div class="alert alert-danger mb-0 text-center" role="alert">
|
||||
{{ message }}
|
||||
</div>
|
||||
<body class="min-h-screen flex flex-col bg-[#F2EFEB] text-[#111111]">
|
||||
|
||||
{% else %}
|
||||
<div class="alert alert-success mb-0 text-center" role="alert">
|
||||
{{ message }}
|
||||
</div>
|
||||
{% endif %}
|
||||
{% endfor %}
|
||||
{% endif %}
|
||||
<nav class="navbar navbar-expand-lg navbar-dark bg-dark">
|
||||
<div class="container">
|
||||
<a href="/" class="navbar-brand d-flex align-items-center">
|
||||
<svg xmlns="http://www.w3.org/2000/svg" width="20" height="20" fill="none" stroke="currentColor"
|
||||
stroke-linecap="round" stroke-linejoin="round" stroke-width="2" aria-hidden="true" class="me-2"
|
||||
viewBox="0 0 24 24">
|
||||
<path d="M23 19a2 2 0 0 1-2 2H3a2 2 0 0 1-2-2V8a2 2 0 0 1 2-2h4l2-3h6l2 3h4a2 2 0 0 1 2 2z" />
|
||||
<circle cx="12" cy="13" r="4" />
|
||||
</svg>
|
||||
<strong>RedditVideoMakerBot</strong>
|
||||
<!-- ── Navigation ─────────────────────────────────────────────────── -->
|
||||
<header class="sticky top-0 z-50 bg-[#F2EFEB] border-b border-[#111111]">
|
||||
<div class="container mx-auto px-4">
|
||||
<div class="flex items-center justify-between h-14">
|
||||
<!-- Logo -->
|
||||
<a href="{{ app_url('/') }}" class="flex items-center gap-2 no-underline">
|
||||
<span class="font-display text-lg text-[#111111] tracking-tighter uppercase">VideoMakerBot</span>
|
||||
</a>
|
||||
|
||||
<div class="collapse navbar-collapse">
|
||||
<ul class="navbar-nav mr-auto">
|
||||
<li class="nav-item">
|
||||
<a class="nav-link" href="backgrounds">Background Manager</a>
|
||||
</li>
|
||||
<li class="nav-item">
|
||||
<a class="nav-link" href="settings">Settings</a>
|
||||
</li>
|
||||
</ul>
|
||||
<!-- Future feature
|
||||
<ul class="navbar-nav">
|
||||
<li class="nav-item">
|
||||
<button class="btn btn-outline-success mr-auto mt-2 mt-lg-0">Create new short</button>
|
||||
</li>
|
||||
</ul>
|
||||
-->
|
||||
<!-- Nav Items -->
|
||||
<div class="flex items-center gap-0">
|
||||
<a href="{{ app_url('/') }}" class="nav-link">Library</a>
|
||||
<a href="{{ app_url('/backgrounds') }}" class="nav-link">Backgrounds</a>
|
||||
<a href="{{ app_url('/settings') }}" class="nav-link">Settings</a>
|
||||
<span class="text-[#111111]/20 mx-1">///</span>
|
||||
<a href="{{ app_url('/create') }}" class="btn-primary-neo text-sm py-1.5 px-4">
|
||||
<i data-lucide="plus" class="w-3.5 h-3.5 inline mr-1"></i>
|
||||
Create
|
||||
</a>
|
||||
</div>
|
||||
</div>
|
||||
</nav>
|
||||
</div>
|
||||
</header>
|
||||
|
||||
{% block main %}{% endblock %}
|
||||
|
||||
<footer class="text-muted py-5">
|
||||
<div class="container">
|
||||
<p class="float-end mb-1">
|
||||
<a href="#">Back to top</a>
|
||||
</p>
|
||||
<p class="mb-1"><a href="https://getbootstrap.com/docs/5.2/examples/album/" target="_blank">Album</a>
|
||||
Example
|
||||
Theme by © Bootstrap. <a
|
||||
href="https://github.com/elebumm/RedditVideoMakerBot/blob/master/README.md#developers-and-maintainers"
|
||||
target="_blank">Developers and Maintainers</a></p>
|
||||
<p class="mb-0">If your data is not refreshing, try to hard reload(Ctrl + F5) or click <a id="hard-reload">this</a> and visit your local
|
||||
|
||||
<strong>{{ file }}</strong> file.
|
||||
</p>
|
||||
<!-- ── Flash Messages ─────────────────────────────────────────────── -->
|
||||
{% if get_flashed_messages() %}
|
||||
<div class="container mx-auto px-4 mt-4">
|
||||
{% for category, message in get_flashed_messages(with_categories=true) %}
|
||||
<div class="border px-4 py-3 mb-2 font-mono text-sm
|
||||
{{ 'border-[#DE6C56] text-[#DE6C56] bg-[#DE6C56]/5' if category == 'error' else 'border-[#4ADE80] text-[#111111] bg-[#4ADE80]/10' }}">
|
||||
<i data-lucide="{{ 'alert-circle' if category == 'error' else 'check-circle' }}" class="w-4 h-4 inline mr-2"></i>
|
||||
<span>{{ message }}</span>
|
||||
</div>
|
||||
{% endfor %}
|
||||
</div>
|
||||
{% endif %}
|
||||
|
||||
<!-- ── Main Content ───────────────────────────────────────────────── -->
|
||||
<main class="flex-grow">
|
||||
{% block main %}{% endblock %}
|
||||
</main>
|
||||
|
||||
<!-- ── Footer Ticker ──────────────────────────────────────────────── -->
|
||||
<footer class="footer-ticker border-t-2 border-[#111111]">
|
||||
<div class="container mx-auto px-4 flex items-center gap-3 overflow-x-auto">
|
||||
<span class="text-[#4ADE80] shrink-0">● ONLINE</span>
|
||||
<span class="text-[#F2EFEB]/30 shrink-0">///</span>
|
||||
<span class="shrink-0">VideoMakerBot</span>
|
||||
<span class="text-[#F2EFEB]/30 shrink-0">///</span>
|
||||
<span class="shrink-0">v3.4.0</span>
|
||||
<span class="text-[#F2EFEB]/30 shrink-0">///</span>
|
||||
<a href="https://github.com/elebumm/RedditVideoMakerBot" target="_blank" class="shrink-0">GITHUB</a>
|
||||
<span class="text-[#F2EFEB]/30 shrink-0">///</span>
|
||||
<a href="#" id="hard-reload" class="shrink-0">RELOAD</a>
|
||||
<span class="text-[#F2EFEB]/30 shrink-0">///</span>
|
||||
<span class="text-[#F2EFEB]/50 shrink-0">© 2026</span>
|
||||
</div>
|
||||
</footer>
|
||||
|
||||
<script>
|
||||
document.getElementById("hard-reload").addEventListener("click", function () {
|
||||
// Initialize Lucide icons
|
||||
lucide.createIcons();
|
||||
|
||||
document.getElementById("hard-reload").addEventListener("click", function (e) {
|
||||
e.preventDefault();
|
||||
window.location.reload(true);
|
||||
});
|
||||
</script>
|
||||
</body>
|
||||
|
||||
</html>
|
||||
</html>
|
||||
|
||||
File diff suppressed because it is too large
Load Diff
@ -1,2 +1,3 @@
|
||||
#!/bin/sh
|
||||
docker build -t rvmt .
|
||||
set -eu
|
||||
docker compose build
|
||||
|
||||
@ -0,0 +1,45 @@
|
||||
services:
|
||||
gui:
|
||||
build:
|
||||
context: .
|
||||
image: videomakerbot:latest
|
||||
command: ["python", "GUI.py"]
|
||||
ports:
|
||||
- "4000:4000"
|
||||
environment:
|
||||
GUI_HOST: "0.0.0.0"
|
||||
GUI_PORT: "4000"
|
||||
GUI_OPEN_BROWSER: "0"
|
||||
GUI_BROWSER_URL: "http://localhost:4000"
|
||||
XDG_CACHE_HOME: "/app/.cache"
|
||||
CLOAKBROWSER_CACHE_DIR: "/app/.cache/cloakbrowser"
|
||||
volumes:
|
||||
- ./:/app
|
||||
shm_size: "1gb"
|
||||
|
||||
cli:
|
||||
build:
|
||||
context: .
|
||||
image: videomakerbot:latest
|
||||
command: ["python", "main.py"]
|
||||
environment:
|
||||
PYTHONUNBUFFERED: "1"
|
||||
XDG_CACHE_HOME: "/app/.cache"
|
||||
CLOAKBROWSER_CACHE_DIR: "/app/.cache/cloakbrowser"
|
||||
volumes:
|
||||
- ./:/app
|
||||
shm_size: "1gb"
|
||||
|
||||
test:
|
||||
build:
|
||||
context: .
|
||||
image: videomakerbot:latest
|
||||
command: ["pytest", "tests/", "-v"]
|
||||
environment:
|
||||
PYTHONUNBUFFERED: "1"
|
||||
PYTHONPATH: "/app"
|
||||
XDG_CACHE_HOME: "/app/.cache"
|
||||
CLOAKBROWSER_CACHE_DIR: "/app/.cache/cloakbrowser"
|
||||
volumes:
|
||||
- ./:/app
|
||||
shm_size: "1gb"
|
||||
@ -0,0 +1,6 @@
|
||||
#!/bin/sh
|
||||
set -eu
|
||||
|
||||
python -m utils.docker_bootstrap
|
||||
|
||||
exec "$@"
|
||||
@ -0,0 +1,29 @@
|
||||
app = "hongphuc-threads-video-maker"
|
||||
primary_region = "sin"
|
||||
|
||||
[build]
|
||||
dockerfile = "Dockerfile"
|
||||
|
||||
[env]
|
||||
GUI_HOST = "0.0.0.0"
|
||||
GUI_PORT = "4000"
|
||||
GUI_OPEN_BROWSER = "0"
|
||||
GUI_BROWSER_URL = "https://hongphuc5497.com/threads-video-maker/"
|
||||
PUBLIC_BASE_PATH = "/threads-video-maker"
|
||||
PUBLIC_DEMO_MODE = "1"
|
||||
PUBLIC_ORIGIN_HOST = "hongphuc5497.com"
|
||||
XDG_CACHE_HOME = "/app/.cache"
|
||||
CLOAKBROWSER_CACHE_DIR = "/app/.cache/cloakbrowser"
|
||||
|
||||
[http_service]
|
||||
internal_port = 4000
|
||||
force_https = true
|
||||
auto_stop_machines = "stop"
|
||||
auto_start_machines = true
|
||||
min_machines_running = 0
|
||||
processes = ["app"]
|
||||
|
||||
[[vm]]
|
||||
memory = "4gb"
|
||||
cpu_kind = "shared"
|
||||
cpus = 2
|
||||
@ -0,0 +1,70 @@
|
||||
"""Platform abstraction layer for content source selection."""
|
||||
|
||||
from utils import settings
|
||||
|
||||
|
||||
def get_content_object(POST_ID=None) -> dict:
|
||||
"""
|
||||
Returns a populated content_object dict for the configured platform.
|
||||
Dispatches to the appropriate platform fetcher based on settings.config["settings"]["platform"].
|
||||
|
||||
Args:
|
||||
POST_ID (str, optional): Specific post ID to fetch. If None, auto-selects a post.
|
||||
|
||||
Returns:
|
||||
dict: Standard content_object with keys:
|
||||
- thread_id, thread_title, thread_url, is_nsfw, thread_category, comments
|
||||
- (or thread_post if storymode is enabled)
|
||||
|
||||
Raises:
|
||||
ValueError: If platform is unknown or invalid.
|
||||
"""
|
||||
platform = settings.config["settings"].get("platform", "reddit").lower()
|
||||
|
||||
if platform == "reddit":
|
||||
from reddit.subreddit import get_subreddit_threads
|
||||
return get_subreddit_threads(POST_ID)
|
||||
|
||||
elif platform == "threads":
|
||||
discovery = settings.config.get("threads", {}).get("discovery_method", "api")
|
||||
if discovery == "scrape":
|
||||
from platforms.threads.scraper import get_trending_threads_content
|
||||
return get_trending_threads_content(POST_ID)
|
||||
else:
|
||||
from platforms.threads.fetcher import get_threads_content
|
||||
return get_threads_content(POST_ID)
|
||||
|
||||
else:
|
||||
raise ValueError(
|
||||
f"Unknown platform: '{platform}'. Valid options: reddit, threads"
|
||||
)
|
||||
|
||||
|
||||
def get_screenshot_fn(platform: str = None):
|
||||
"""
|
||||
Returns the appropriate screenshot function for the given platform.
|
||||
|
||||
Args:
|
||||
platform (str, optional): Platform name. If None, uses the configured platform.
|
||||
|
||||
Returns:
|
||||
callable: Screenshot function that takes (content_object, screenshot_num).
|
||||
|
||||
Raises:
|
||||
ValueError: If platform is unknown or invalid.
|
||||
"""
|
||||
if platform is None:
|
||||
platform = settings.config["settings"].get("platform", "reddit").lower()
|
||||
|
||||
if platform == "reddit":
|
||||
from video_creation.screenshot_downloader import get_screenshots_of_reddit_posts
|
||||
return get_screenshots_of_reddit_posts
|
||||
|
||||
elif platform == "threads":
|
||||
from platforms.threads.screenshot import get_screenshots_of_threads_posts
|
||||
return get_screenshots_of_threads_posts
|
||||
|
||||
else:
|
||||
raise ValueError(
|
||||
f"Unknown platform: '{platform}'. Valid options: reddit, threads"
|
||||
)
|
||||
@ -0,0 +1 @@
|
||||
"""Threads (Meta) platform integration for VideoMakerBot."""
|
||||
@ -0,0 +1,176 @@
|
||||
"""Shared Playwright authentication for Threads.
|
||||
|
||||
Used by both the screenshotter (screenshot.py) and the web scraper (scraper.py).
|
||||
"""
|
||||
|
||||
import json
|
||||
from pathlib import Path
|
||||
from urllib.parse import urlparse
|
||||
|
||||
from playwright.sync_api import Browser, BrowserContext, Page, TimeoutError, ViewportSize
|
||||
|
||||
from utils import settings
|
||||
from utils.console import emit_scraper_event, print_substep
|
||||
|
||||
THREADS_LOGIN_URL = "https://www.threads.com/login"
|
||||
THREADS_AUTH_CHECK_URL = "https://www.threads.net/"
|
||||
THREADS_COOKIE_FILE = "./video_creation/data/cookie-threads.json"
|
||||
DEFAULT_USER_AGENT = (
|
||||
"Mozilla/5.0 (Windows NT 10.0; Win64; x64) "
|
||||
"AppleWebKit/537.36 (KHTML, like Gecko) "
|
||||
"Chrome/120.0.0.0 Safari/537.36"
|
||||
)
|
||||
THREADS_HOSTS = {
|
||||
"threads.com",
|
||||
"www.threads.com",
|
||||
"threads.net",
|
||||
"www.threads.net",
|
||||
}
|
||||
THREADS_NON_AUTH_PATH_PREFIXES = (
|
||||
"/login",
|
||||
"/challenge",
|
||||
"/checkpoint",
|
||||
"/consent",
|
||||
"/accountsuspended",
|
||||
"/suspended",
|
||||
)
|
||||
|
||||
|
||||
def _is_logged_in_threads_url(url: str) -> bool:
|
||||
"""Return True once Threads has navigated away from the login page."""
|
||||
parsed = urlparse(url)
|
||||
hostname = (parsed.hostname or "").casefold()
|
||||
path = (parsed.path or "/").lower()
|
||||
return parsed.scheme == "https" and hostname in THREADS_HOSTS and not any(
|
||||
path.startswith(prefix) for prefix in THREADS_NON_AUTH_PATH_PREFIXES
|
||||
)
|
||||
|
||||
|
||||
def _has_login_gate(page: Page) -> bool:
|
||||
"""Detect Threads login modal overlaying the page."""
|
||||
gate_selectors = [
|
||||
'input[autocomplete="username"]',
|
||||
'input[autocomplete="current-password"]',
|
||||
'div[role="dialog"] button:has-text("Log in")',
|
||||
]
|
||||
for selector in gate_selectors:
|
||||
try:
|
||||
if page.locator(selector).first.is_visible():
|
||||
return True
|
||||
except Exception:
|
||||
continue
|
||||
return False
|
||||
|
||||
|
||||
def login_to_threads(page: Page, _context: BrowserContext) -> None:
|
||||
"""Log into Threads via Instagram credentials and persist session cookies."""
|
||||
username = settings.config["threads"]["creds"].get("username", "").strip()
|
||||
password = settings.config["threads"]["creds"].get("password", "").strip()
|
||||
|
||||
if not username or not password:
|
||||
raise RuntimeError(
|
||||
"Threads login requires credentials. "
|
||||
"Set threads.creds.username and threads.creds.password in config.toml"
|
||||
)
|
||||
|
||||
print_substep("Logging into Threads (via Instagram)...")
|
||||
emit_scraper_event("login", {"message": "Logging into Threads (via Instagram)..."})
|
||||
page.goto(THREADS_LOGIN_URL, timeout=0)
|
||||
page.wait_for_load_state("domcontentloaded")
|
||||
|
||||
username_input = page.locator('input[autocomplete="username"]')
|
||||
password_input = page.locator('input[autocomplete="current-password"]')
|
||||
username_input.fill(username)
|
||||
password_input.fill(password)
|
||||
|
||||
# CloakBrowser can be slightly flaky with the humanized button click even
|
||||
# when the credentials are correct. Try the visible login button first,
|
||||
# then fall back to pressing Enter in the password field.
|
||||
submit_attempts = [
|
||||
("button click", lambda: page.get_by_role("button", name="Log in", exact=True).first.click()),
|
||||
("password Enter", lambda: password_input.press("Enter")),
|
||||
]
|
||||
last_error = None
|
||||
for index, (label, submit) in enumerate(submit_attempts):
|
||||
try:
|
||||
submit()
|
||||
# Threads currently redirects to threads.com on successful login,
|
||||
# but older saved sessions and some flows may still hit threads.net.
|
||||
page.wait_for_url(_is_logged_in_threads_url, timeout=30000 if index == 0 else 15000)
|
||||
break
|
||||
except TimeoutError as error:
|
||||
last_error = error
|
||||
if index == len(submit_attempts) - 1:
|
||||
raise
|
||||
print_substep(f"Threads login via {label} timed out. Retrying...", style="yellow")
|
||||
|
||||
page.wait_for_load_state("domcontentloaded")
|
||||
|
||||
cookies = _context.cookies()
|
||||
Path(THREADS_COOKIE_FILE).parent.mkdir(parents=True, exist_ok=True)
|
||||
with open(THREADS_COOKIE_FILE, "w") as f:
|
||||
json.dump(cookies, f)
|
||||
|
||||
print_substep("Logged into Threads and saved session cookies.", style="bold green")
|
||||
emit_scraper_event("login", {"message": "Logged in successfully"})
|
||||
|
||||
|
||||
def ensure_authenticated_context(browser: Browser, **kwargs) -> BrowserContext:
|
||||
"""Create a Playwright browser context with Threads session cookies loaded.
|
||||
|
||||
Loads saved cookies from cookie-threads.json. If no valid session exists,
|
||||
performs a fresh login and persists the cookies.
|
||||
|
||||
Keyword arguments override defaults for locale, viewport, device_scale_factor,
|
||||
color_scheme, and user_agent.
|
||||
"""
|
||||
theme = settings.config["settings"]["theme"]
|
||||
W = int(settings.config["settings"]["resolution_w"])
|
||||
H = int(settings.config["settings"]["resolution_h"])
|
||||
dsf = (W // 600) + 1
|
||||
|
||||
defaults = {
|
||||
"locale": "en-US",
|
||||
"color_scheme": "dark" if theme == "dark" else "light",
|
||||
"viewport": ViewportSize(width=W, height=H),
|
||||
"device_scale_factor": dsf,
|
||||
"user_agent": DEFAULT_USER_AGENT,
|
||||
}
|
||||
defaults.update(kwargs)
|
||||
|
||||
context = browser.new_context(**defaults)
|
||||
|
||||
cookie_path = Path(THREADS_COOKIE_FILE)
|
||||
if cookie_path.exists():
|
||||
try:
|
||||
with open(cookie_path, encoding="utf-8") as f:
|
||||
saved_cookies = json.load(f)
|
||||
context.add_cookies(saved_cookies)
|
||||
print_substep("Loaded saved Threads session cookies.")
|
||||
emit_scraper_event("login", {"message": "Loaded saved session cookies"})
|
||||
|
||||
# Verify loaded cookies are still valid
|
||||
page = context.new_page()
|
||||
try:
|
||||
page.goto(THREADS_AUTH_CHECK_URL, timeout=0)
|
||||
page.wait_for_load_state("domcontentloaded")
|
||||
page.wait_for_timeout(1500)
|
||||
if _has_login_gate(page):
|
||||
print_substep("Saved Threads cookies expired. Re-logging in...", style="yellow")
|
||||
context.clear_cookies()
|
||||
cookie_path.unlink(missing_ok=True)
|
||||
login_to_threads(page, context)
|
||||
finally:
|
||||
page.close()
|
||||
except (json.JSONDecodeError, IOError):
|
||||
print_substep("Saved cookies corrupted. Logging in fresh...")
|
||||
page = context.new_page()
|
||||
login_to_threads(page, context)
|
||||
page.close()
|
||||
else:
|
||||
print_substep("No saved cookies found. Logging in...")
|
||||
page = context.new_page()
|
||||
login_to_threads(page, context)
|
||||
page.close()
|
||||
|
||||
return context
|
||||
@ -0,0 +1,190 @@
|
||||
"""Fetches content from Meta Threads via the Graph API."""
|
||||
|
||||
import requests
|
||||
from typing import Optional
|
||||
|
||||
from utils import settings
|
||||
from utils.console import print_step, print_substep
|
||||
from utils.voice import sanitize_text
|
||||
from utils.videos import check_done_by_id
|
||||
|
||||
|
||||
GRAPH_API_BASE = "https://graph.threads.net/v1.0"
|
||||
|
||||
|
||||
def _get_headers() -> dict:
|
||||
"""Returns HTTP headers with Bearer token for Graph API requests."""
|
||||
token = settings.config["threads"]["creds"]["access_token"]
|
||||
if not token:
|
||||
raise RuntimeError(
|
||||
"Threads API: access_token is required. "
|
||||
"Set it in config.toml under [threads.creds]."
|
||||
)
|
||||
return {"Authorization": f"Bearer {token}"}
|
||||
|
||||
|
||||
def _api_get(url: str, params: dict = None) -> dict:
|
||||
"""Makes a GET request to Threads Graph API with error handling."""
|
||||
try:
|
||||
resp = requests.get(url, headers=_get_headers(), params=params or {}, timeout=15)
|
||||
resp.raise_for_status()
|
||||
return resp.json()
|
||||
except requests.exceptions.HTTPError as e:
|
||||
if e.response.status_code == 401:
|
||||
raise RuntimeError(
|
||||
"Threads API: Invalid or expired access_token. "
|
||||
"Tokens are valid for 60 days. Refresh at: "
|
||||
"https://developers.facebook.com/tools/explorer/"
|
||||
) from e
|
||||
if e.response.status_code == 400:
|
||||
error_msg = e.response.json().get("error", {}).get("message", str(e))
|
||||
raise RuntimeError(f"Threads API: Bad request — {error_msg}") from e
|
||||
raise RuntimeError(f"Threads API: HTTP {e.response.status_code}") from e
|
||||
except requests.exceptions.ConnectionError as e:
|
||||
raise RuntimeError("Threads API: Cannot connect. Check internet connection.") from e
|
||||
except requests.exceptions.Timeout as e:
|
||||
raise RuntimeError("Threads API: Request timed out.") from e
|
||||
|
||||
|
||||
def _fetch_post(post_id: str) -> dict:
|
||||
"""Fetches a single Threads post by ID."""
|
||||
url = f"{GRAPH_API_BASE}/{post_id}"
|
||||
params = {"fields": "id,text,timestamp,permalink,is_quote_post,media_type"}
|
||||
return _api_get(url, params)
|
||||
|
||||
|
||||
def _fetch_replies(post_id: str, limit: int = 50) -> list:
|
||||
"""Fetches all replies to a Threads post, handling pagination."""
|
||||
url = f"{GRAPH_API_BASE}/{post_id}/replies"
|
||||
params = {
|
||||
"fields": "id,text,timestamp,username,permalink",
|
||||
"limit": limit,
|
||||
}
|
||||
results = []
|
||||
|
||||
while url:
|
||||
data = _api_get(url, params)
|
||||
results.extend(data.get("data", []))
|
||||
# Handle pagination — next URL is provided in paging.next
|
||||
url = data.get("paging", {}).get("next")
|
||||
params = {} # Next URL already includes all params
|
||||
|
||||
return results
|
||||
|
||||
|
||||
def _pick_best_post() -> tuple:
|
||||
"""
|
||||
Fetches recent posts from the user and returns the first one
|
||||
with enough replies that hasn't been processed yet.
|
||||
|
||||
Returns:
|
||||
tuple: (post_dict, replies_list)
|
||||
|
||||
Raises:
|
||||
RuntimeError: If no eligible posts are found.
|
||||
"""
|
||||
user_id = settings.config["threads"]["creds"]["user_id"]
|
||||
if not user_id:
|
||||
raise RuntimeError(
|
||||
"Threads API: user_id is required. "
|
||||
"Set it in config.toml under [threads.creds]."
|
||||
)
|
||||
|
||||
url = f"{GRAPH_API_BASE}/{user_id}/threads"
|
||||
params = {"fields": "id,text,timestamp,permalink,media_type", "limit": 25}
|
||||
|
||||
data = _api_get(url, params)
|
||||
posts = data.get("data", [])
|
||||
|
||||
min_replies = settings.config["threads"]["thread"]["min_replies"]
|
||||
|
||||
for post in posts:
|
||||
if check_done_by_id(post["id"]):
|
||||
continue
|
||||
|
||||
replies = _fetch_replies(post["id"])
|
||||
if len(replies) >= min_replies:
|
||||
return post, replies
|
||||
|
||||
raise RuntimeError(
|
||||
f"No eligible Threads posts found. "
|
||||
f"Ensure you have posts with at least {min_replies} replies."
|
||||
)
|
||||
|
||||
|
||||
def get_threads_content(POST_ID: str = None) -> dict:
|
||||
"""
|
||||
Fetches Threads content (post + replies) and returns it in the standard content_object format.
|
||||
|
||||
Args:
|
||||
POST_ID (str, optional): Specific post ID to fetch. If None, auto-selects.
|
||||
|
||||
Returns:
|
||||
dict: Standard content_object matching the pipeline contract.
|
||||
|
||||
Raises:
|
||||
RuntimeError: On API errors or if no eligible content found.
|
||||
"""
|
||||
print_step("Fetching Threads content...")
|
||||
|
||||
# Determine which post to fetch
|
||||
if POST_ID:
|
||||
post = _fetch_post(POST_ID)
|
||||
replies = _fetch_replies(POST_ID)
|
||||
elif settings.config["threads"]["thread"].get("post_id"):
|
||||
post_id = settings.config["threads"]["thread"]["post_id"]
|
||||
post = _fetch_post(post_id)
|
||||
replies = _fetch_replies(post_id)
|
||||
else:
|
||||
post, replies = _pick_best_post()
|
||||
|
||||
# Load content filters from config
|
||||
max_len = settings.config["threads"]["thread"]["max_reply_length"]
|
||||
min_len = settings.config["threads"]["thread"]["min_reply_length"]
|
||||
blocked_raw = settings.config["threads"]["thread"].get("blocked_words", "")
|
||||
blocked = [w.strip().lower() for w in blocked_raw.split(",") if w.strip()]
|
||||
|
||||
# Build content object in standard format
|
||||
content = {
|
||||
"thread_id": post["id"],
|
||||
"thread_title": (post.get("text") or "")[:280], # Threads has no separate title
|
||||
"thread_url": post["permalink"],
|
||||
"is_nsfw": False, # Threads API doesn't provide NSFW flag
|
||||
"thread_category": "threads", # Generic field for output folder naming
|
||||
"comments": [],
|
||||
}
|
||||
|
||||
# Filter and add replies
|
||||
for reply in replies:
|
||||
body = reply.get("text", "").strip()
|
||||
if not body:
|
||||
continue
|
||||
|
||||
# Check blocked words
|
||||
if any(w in body.lower() for w in blocked):
|
||||
continue
|
||||
|
||||
# Check length constraints
|
||||
if not (min_len <= len(body) <= max_len):
|
||||
continue
|
||||
|
||||
# Sanitize text
|
||||
sanitised = sanitize_text(body)
|
||||
if not sanitised:
|
||||
continue
|
||||
|
||||
content["comments"].append({
|
||||
"comment_body": body,
|
||||
"comment_url": reply["permalink"],
|
||||
"comment_id": reply["id"],
|
||||
})
|
||||
|
||||
# Log summary
|
||||
title_preview = content["thread_title"][:60]
|
||||
print_substep(
|
||||
f"Fetched Threads post '{title_preview}...' "
|
||||
f"with {len(content['comments'])} replies.",
|
||||
style="bold green",
|
||||
)
|
||||
|
||||
return content
|
||||
@ -0,0 +1,644 @@
|
||||
"""Web scraping-based trending post discovery for Threads.net.
|
||||
|
||||
Bypasses the Meta Graph API (which only accesses your own posts) by using Playwright
|
||||
to scrape threads.net directly — the "For You" feed, post pages, and replies.
|
||||
Returns the standard content_object dict consumed by the rest of the pipeline.
|
||||
"""
|
||||
|
||||
import re
|
||||
from typing import Optional
|
||||
|
||||
from playwright.sync_api import BrowserContext, Locator
|
||||
|
||||
from platforms.threads.auth import ensure_authenticated_context
|
||||
from utils import settings
|
||||
from utils.browser_backend import launch_browser
|
||||
from utils.console import emit_scraper_event, print_step, print_substep
|
||||
from utils.voice import sanitize_text
|
||||
from utils.videos import check_done_by_id
|
||||
|
||||
FEED_URL = "https://www.threads.net"
|
||||
SCROLL_DELAY_MS = 2000
|
||||
MAX_FEED_SCROLLS = 36
|
||||
POST_LINK_SELECTOR = 'a[href*="/post/"]'
|
||||
CARD_XPATH = 'xpath=ancestor::div[contains(@class, "x1a2a7pz")][1]'
|
||||
|
||||
|
||||
def _post_id_from_url(url: str) -> str:
|
||||
return url.rstrip("/").split("/")[-1]
|
||||
|
||||
|
||||
def _to_absolute_url(href: str) -> str:
|
||||
if href.startswith("http"):
|
||||
return href
|
||||
return "https://www.threads.net" + href
|
||||
|
||||
|
||||
def _parse_abbreviated_number(s: str) -> int:
|
||||
"""Parse abbreviated numbers like '1K', '2.5M' into integers."""
|
||||
s = s.strip().upper().replace(",", "")
|
||||
if not s:
|
||||
return 0
|
||||
multipliers = {"K": 1_000, "M": 1_000_000}
|
||||
if s[-1] in multipliers:
|
||||
try:
|
||||
return int(float(s[:-1]) * multipliers[s[-1]])
|
||||
except ValueError:
|
||||
return 0
|
||||
try:
|
||||
return int(s)
|
||||
except ValueError:
|
||||
return 0
|
||||
|
||||
|
||||
def _parse_card_text(text: str) -> dict:
|
||||
"""Parse a Threads card's raw text into structured data.
|
||||
|
||||
Threads card format:
|
||||
line 0: username
|
||||
line 1: timestamp (e.g. "14h", "1d")
|
||||
lines 2..N: post body text
|
||||
last 1-4 lines: engagement metrics (likes, replies, reposts, quotes)
|
||||
|
||||
Returns dict with keys: username, timestamp, body, likes, replies, reposts
|
||||
"""
|
||||
if not text:
|
||||
return {"username": "", "timestamp": "", "body": "", "likes": 0, "replies": 0, "reposts": 0}
|
||||
|
||||
lines = text.strip().split("\n")
|
||||
if len(lines) < 3:
|
||||
return {"username": "", "timestamp": "", "body": text, "likes": 0, "replies": 0, "reposts": 0}
|
||||
|
||||
username = lines[0].strip()
|
||||
timestamp = lines[1].strip()
|
||||
|
||||
# Find where engagement metrics start (trailing numeric/abbreviated lines)
|
||||
metric_start = len(lines)
|
||||
for i in range(len(lines) - 1, 1, -1):
|
||||
line = lines[i].strip()
|
||||
if re.match(r'^[\d,.]+[KkMm]?$', line):
|
||||
metric_start = i
|
||||
else:
|
||||
break
|
||||
|
||||
# Body is everything between timestamp and metrics
|
||||
body_lines = lines[2:metric_start]
|
||||
body = "\n".join(body_lines).strip()
|
||||
|
||||
# Parse engagement metrics from the end
|
||||
metrics = lines[metric_start:]
|
||||
likes = 0
|
||||
replies_count = 0
|
||||
reposts = 0
|
||||
|
||||
if len(metrics) >= 1:
|
||||
likes = _parse_abbreviated_number(metrics[0])
|
||||
if len(metrics) >= 2:
|
||||
replies_count = _parse_abbreviated_number(metrics[1])
|
||||
if len(metrics) >= 3:
|
||||
reposts = _parse_abbreviated_number(metrics[2])
|
||||
|
||||
return {
|
||||
"username": username,
|
||||
"timestamp": timestamp,
|
||||
"body": body,
|
||||
"likes": likes,
|
||||
"replies": replies_count,
|
||||
"reposts": reposts,
|
||||
}
|
||||
|
||||
|
||||
def _extract_text_from_card(link: Locator) -> str:
|
||||
"""Walk up from a post link to the card container and extract its raw text."""
|
||||
try:
|
||||
card = link.locator(CARD_XPATH)
|
||||
if card.count():
|
||||
return card.first.inner_text(timeout=3000).strip()
|
||||
except Exception:
|
||||
pass
|
||||
return ""
|
||||
|
||||
|
||||
# --- Feed scraping ---
|
||||
|
||||
|
||||
def _scrape_feed_posts(context: BrowserContext, max_scrolls: int = MAX_FEED_SCROLLS) -> list[dict]:
|
||||
"""Navigate to threads.net feed, scroll, extract post metadata with engagement metrics."""
|
||||
print_step("Scraping Threads trending feed...")
|
||||
emit_scraper_event("browser_launch", {"message": "Scraping Threads trending feed"})
|
||||
page = context.new_page()
|
||||
posts: list[dict] = []
|
||||
seen_ids: set[str] = set()
|
||||
|
||||
try:
|
||||
page.goto(FEED_URL, timeout=0)
|
||||
page.wait_for_timeout(4000)
|
||||
|
||||
last_height = 0
|
||||
|
||||
for i in range(max_scrolls):
|
||||
links = page.locator(POST_LINK_SELECTOR).all()
|
||||
new_found = 0
|
||||
|
||||
for link in links:
|
||||
href = link.get_attribute("href")
|
||||
if not href:
|
||||
continue
|
||||
post_id = _post_id_from_url(href)
|
||||
if post_id in seen_ids:
|
||||
continue
|
||||
seen_ids.add(post_id)
|
||||
|
||||
raw_text = _extract_text_from_card(link)
|
||||
parsed = _parse_card_text(raw_text)
|
||||
|
||||
posts.append({
|
||||
"url": _to_absolute_url(href),
|
||||
"text": raw_text,
|
||||
"body": parsed["body"],
|
||||
"username": parsed["username"],
|
||||
"timestamp": parsed["timestamp"],
|
||||
"likes": parsed["likes"],
|
||||
"replies_shown": parsed["replies"],
|
||||
"reposts": parsed["reposts"],
|
||||
"post_id": post_id,
|
||||
})
|
||||
new_found += 1
|
||||
|
||||
emit_scraper_event("post_discovered", {
|
||||
"username": parsed["username"],
|
||||
"body": parsed["body"][:100],
|
||||
"likes": parsed["likes"],
|
||||
"replies": parsed["replies"],
|
||||
"reposts": parsed["reposts"],
|
||||
"post_id": post_id,
|
||||
})
|
||||
|
||||
if new_found > 0:
|
||||
top = posts[-1]
|
||||
print_substep(
|
||||
f"Scroll {i + 1}: +{new_found} posts | top: "
|
||||
f"♥{top['likes']:,} 💬{top['replies_shown']} 🔁{top['reposts']} "
|
||||
f"'{top['body'][:50]}...'",
|
||||
style="dim",
|
||||
)
|
||||
|
||||
emit_scraper_event("feed_scroll", {
|
||||
"scroll": i + 1,
|
||||
"new_posts": new_found,
|
||||
"total_posts": len(posts),
|
||||
"max_scrolls": max_scrolls,
|
||||
})
|
||||
|
||||
if new_found == 0 and i > 5:
|
||||
break
|
||||
|
||||
page.evaluate("window.scrollBy(0, document.body.scrollHeight)")
|
||||
page.wait_for_timeout(SCROLL_DELAY_MS)
|
||||
|
||||
new_height = page.evaluate("document.body.scrollHeight")
|
||||
if new_height == last_height:
|
||||
break
|
||||
last_height = new_height
|
||||
|
||||
finally:
|
||||
page.close()
|
||||
|
||||
print_substep(f"Scraped {len(posts)} posts from feed.", style="bold green")
|
||||
return posts
|
||||
|
||||
|
||||
def _scrape_search_page(context: BrowserContext, query: str, max_scrolls: int = 5) -> list[dict]:
|
||||
"""Search Threads for a query and scrape the results.
|
||||
|
||||
Uses the same card extraction as the main feed.
|
||||
"""
|
||||
print_step(f"Scraping Threads search: '{query}'...")
|
||||
emit_scraper_event("search_query", {"query": query, "posts_found": 0})
|
||||
page = context.new_page()
|
||||
posts: list[dict] = []
|
||||
seen_ids: set[str] = set()
|
||||
search_url = f"https://www.threads.net/search?q={query}&serp_type=tags"
|
||||
|
||||
try:
|
||||
page.goto(search_url, timeout=0)
|
||||
page.wait_for_timeout(4000)
|
||||
|
||||
for i in range(max_scrolls):
|
||||
links = page.locator(POST_LINK_SELECTOR).all()
|
||||
new_found = 0
|
||||
|
||||
for link in links:
|
||||
href = link.get_attribute("href")
|
||||
if not href:
|
||||
continue
|
||||
post_id = _post_id_from_url(href)
|
||||
if post_id in seen_ids:
|
||||
continue
|
||||
seen_ids.add(post_id)
|
||||
|
||||
raw_text = _extract_text_from_card(link)
|
||||
parsed = _parse_card_text(raw_text)
|
||||
|
||||
posts.append({
|
||||
"url": _to_absolute_url(href),
|
||||
"text": raw_text,
|
||||
"body": parsed["body"],
|
||||
"username": parsed["username"],
|
||||
"timestamp": parsed["timestamp"],
|
||||
"likes": parsed["likes"],
|
||||
"replies_shown": parsed["replies"],
|
||||
"reposts": parsed["reposts"],
|
||||
"post_id": post_id,
|
||||
})
|
||||
new_found += 1
|
||||
|
||||
if new_found == 0:
|
||||
break
|
||||
|
||||
page.evaluate("window.scrollBy(0, document.body.scrollHeight)")
|
||||
page.wait_for_timeout(SCROLL_DELAY_MS)
|
||||
|
||||
finally:
|
||||
page.close()
|
||||
|
||||
print_substep(f"Search '{query}': {len(posts)} posts.", style="dim")
|
||||
emit_scraper_event("search_query", {"query": query, "posts_found": len(posts)})
|
||||
return posts
|
||||
|
||||
|
||||
# --- Candidate filtering ---
|
||||
|
||||
|
||||
def _parse_timestamp_to_hours(ts: str) -> float | None:
|
||||
"""Convert a Threads timestamp like '14h', '1d', '3d' to hours.
|
||||
|
||||
Returns None if the format is unrecognized.
|
||||
"""
|
||||
if not ts:
|
||||
return None
|
||||
ts = ts.strip().lower()
|
||||
if ts.endswith("h"):
|
||||
try:
|
||||
return float(ts[:-1])
|
||||
except ValueError:
|
||||
return None
|
||||
elif ts.endswith("d"):
|
||||
try:
|
||||
return float(ts[:-1]) * 24
|
||||
except ValueError:
|
||||
return None
|
||||
elif ts.endswith("w"):
|
||||
try:
|
||||
return float(ts[:-1]) * 24 * 7
|
||||
except ValueError:
|
||||
return None
|
||||
elif ts.endswith("m") and not ts.endswith("min"):
|
||||
try:
|
||||
return float(ts[:-1]) * 24 * 30
|
||||
except ValueError:
|
||||
return None
|
||||
return None
|
||||
|
||||
|
||||
def _age_from_config() -> float | None:
|
||||
"""Parse max_post_age config value into hours. Returns None if disabled."""
|
||||
raw = settings.config["threads"]["thread"].get("max_post_age", "")
|
||||
if not raw:
|
||||
return None
|
||||
return _parse_timestamp_to_hours(raw)
|
||||
|
||||
|
||||
def _contains_blocked(text: str, blocked_raw: str) -> bool:
|
||||
if not blocked_raw:
|
||||
return False
|
||||
blocked = [w.strip().lower() for w in blocked_raw.split(",") if w.strip()]
|
||||
text_lower = text.lower()
|
||||
return any(word in text_lower for word in blocked)
|
||||
|
||||
|
||||
def _is_english_text(text: str) -> bool:
|
||||
"""Return True if text is predominantly English (Latin alphabet)."""
|
||||
if not text or not text.strip():
|
||||
return False
|
||||
alpha_chars = [c for c in text if c.isalpha()]
|
||||
if not alpha_chars:
|
||||
return False
|
||||
latin = sum(1 for c in alpha_chars if "a" <= c.lower() <= "z")
|
||||
return latin / len(alpha_chars) >= 0.70
|
||||
|
||||
|
||||
def _filter_candidates(posts: list[dict]) -> list[dict]:
|
||||
"""Filter feed posts by engagement, blocked words, and duplicates.
|
||||
|
||||
Sorts by total engagement (likes + replies) descending so the most
|
||||
viral posts are tried first.
|
||||
"""
|
||||
t_config = settings.config["threads"]["thread"]
|
||||
blocked_raw = t_config.get("blocked_words", "")
|
||||
min_engagement = int(t_config.get("min_engagement", 0))
|
||||
|
||||
max_age_hours = _age_from_config()
|
||||
|
||||
candidates = []
|
||||
for post in posts:
|
||||
if check_done_by_id(post["post_id"]):
|
||||
continue
|
||||
if _contains_blocked(post["body"], blocked_raw):
|
||||
continue
|
||||
if not post["body"] or len(post["body"].strip()) < 10:
|
||||
continue
|
||||
if t_config.get("english_only", False) and not _is_english_text(post["body"]):
|
||||
continue
|
||||
# Age filter
|
||||
if max_age_hours is not None:
|
||||
post_hours = _parse_timestamp_to_hours(post.get("timestamp", ""))
|
||||
if post_hours is not None and post_hours > max_age_hours:
|
||||
continue
|
||||
total_engagement = post.get("likes", 0) + post.get("reposts", 0)
|
||||
if total_engagement < min_engagement:
|
||||
continue
|
||||
post["_total_engagement"] = total_engagement
|
||||
candidates.append(post)
|
||||
|
||||
# Sort by engagement descending — most viral first
|
||||
candidates.sort(key=lambda p: p.get("_total_engagement", 0), reverse=True)
|
||||
|
||||
emit_scraper_event("filter_results", {
|
||||
"before": len(posts),
|
||||
"after": len(candidates),
|
||||
"min_engagement": min_engagement,
|
||||
"max_age_hours": max_age_hours,
|
||||
})
|
||||
|
||||
age_str = f", max age ≤{max_age_hours}h" if max_age_hours else ""
|
||||
if min_engagement > 0:
|
||||
print_substep(
|
||||
f"Filtered {len(posts)} posts -> {len(candidates)} viral candidates "
|
||||
f"(min ♥+🔁 ≥ {min_engagement:,}{age_str})",
|
||||
style="dim",
|
||||
)
|
||||
else:
|
||||
print_substep(
|
||||
f"Filtered {len(posts)} posts -> {len(candidates)} candidates"
|
||||
f"{' (max age ≤' + str(max_age_hours) + 'h)' if max_age_hours else ''}",
|
||||
style="dim",
|
||||
)
|
||||
return candidates
|
||||
|
||||
|
||||
# --- Reply scraping on post pages ---
|
||||
|
||||
|
||||
def _scrape_post_replies(context: BrowserContext, post_url: str, max_replies: int = 100) -> list[dict]:
|
||||
"""Navigate to a post page, scroll to load replies, extract reply data.
|
||||
|
||||
Uses _parse_card_text to separate reply body from metadata (username, timestamp, etc.).
|
||||
"""
|
||||
page = context.new_page()
|
||||
replies: list[dict] = []
|
||||
seen_ids: set[str] = set()
|
||||
main_post_id = _post_id_from_url(post_url)
|
||||
|
||||
try:
|
||||
page.goto(post_url, timeout=0)
|
||||
page.wait_for_timeout(4000)
|
||||
|
||||
stable_count = 0
|
||||
last_count = 0
|
||||
|
||||
for _ in range(15):
|
||||
links = page.locator(POST_LINK_SELECTOR).all()
|
||||
|
||||
for link in links:
|
||||
href = link.get_attribute("href")
|
||||
if not href:
|
||||
continue
|
||||
reply_id = _post_id_from_url(href)
|
||||
if reply_id == main_post_id:
|
||||
continue
|
||||
if reply_id in seen_ids:
|
||||
continue
|
||||
seen_ids.add(reply_id)
|
||||
|
||||
raw_text = _extract_text_from_card(link)
|
||||
if not raw_text:
|
||||
continue
|
||||
|
||||
parsed = _parse_card_text(raw_text)
|
||||
cleaned_body = parsed["body"]
|
||||
|
||||
replies.append({
|
||||
"comment_body": cleaned_body,
|
||||
"comment_url": _to_absolute_url(href),
|
||||
"comment_id": reply_id,
|
||||
})
|
||||
|
||||
if len(replies) >= max_replies:
|
||||
break
|
||||
|
||||
if len(replies) >= max_replies:
|
||||
break
|
||||
|
||||
if len(replies) == last_count:
|
||||
stable_count += 1
|
||||
if stable_count >= 3:
|
||||
break
|
||||
else:
|
||||
stable_count = 0
|
||||
last_count = len(replies)
|
||||
|
||||
page.evaluate("window.scrollBy(0, document.body.scrollHeight)")
|
||||
page.wait_for_timeout(1500)
|
||||
|
||||
finally:
|
||||
page.close()
|
||||
|
||||
return replies
|
||||
|
||||
|
||||
def _scrape_main_post_text(context: BrowserContext, post_url: str) -> str:
|
||||
"""Extract and clean the main post text from a post page."""
|
||||
page = context.new_page()
|
||||
try:
|
||||
page.goto(post_url, timeout=0)
|
||||
page.wait_for_timeout(3000)
|
||||
|
||||
links = page.locator(POST_LINK_SELECTOR).all()
|
||||
for link in links:
|
||||
href = link.get_attribute("href")
|
||||
if href and _post_id_from_url(href) == _post_id_from_url(post_url):
|
||||
raw = _extract_text_from_card(link)
|
||||
if raw:
|
||||
parsed = _parse_card_text(raw)
|
||||
return parsed["body"] or raw
|
||||
return ""
|
||||
finally:
|
||||
page.close()
|
||||
|
||||
|
||||
# --- Content object builder ---
|
||||
|
||||
|
||||
def _build_content_object(post: dict, replies: list[dict]) -> dict:
|
||||
"""Build the standard content_object from scraped post + replies.
|
||||
|
||||
Uses cleaned body text for title and comment bodies.
|
||||
"""
|
||||
t_config = settings.config["threads"]["thread"]
|
||||
max_len = int(t_config["max_reply_length"])
|
||||
min_len = int(t_config["min_reply_length"])
|
||||
blocked_raw = t_config.get("blocked_words", "")
|
||||
|
||||
storymode = settings.config["settings"].get("storymode", False)
|
||||
|
||||
# Use cleaned body text for the title, fall back to raw text
|
||||
title = post.get("body") or post.get("text") or ""
|
||||
|
||||
content: dict = {
|
||||
"thread_id": post["post_id"],
|
||||
"thread_title": title[:280],
|
||||
"thread_url": post["url"],
|
||||
"is_nsfw": False,
|
||||
"thread_category": "threads",
|
||||
"comments": [],
|
||||
}
|
||||
|
||||
if storymode:
|
||||
content["thread_post"] = title
|
||||
print_substep("Storymode: using post text as thread_post.", style="dim")
|
||||
return content
|
||||
|
||||
for reply in replies:
|
||||
body = reply.get("comment_body", "").strip()
|
||||
if not body:
|
||||
continue
|
||||
if _contains_blocked(body, blocked_raw):
|
||||
continue
|
||||
if t_config.get("english_only", False) and not _is_english_text(body):
|
||||
continue
|
||||
if not (min_len <= len(body) <= max_len):
|
||||
continue
|
||||
sanitised = sanitize_text(body)
|
||||
if not sanitised:
|
||||
continue
|
||||
|
||||
content["comments"].append({
|
||||
"comment_body": body,
|
||||
"comment_url": reply["comment_url"],
|
||||
"comment_id": reply["comment_id"],
|
||||
})
|
||||
|
||||
return content
|
||||
|
||||
|
||||
# --- Main entry point ---
|
||||
|
||||
|
||||
def get_trending_threads_content(POST_ID: Optional[str] = None) -> dict:
|
||||
"""Discover trending Threads posts via web scraping and return a content_object."""
|
||||
print_step("Discovering trending Threads content via web scraping...")
|
||||
|
||||
min_replies = int(settings.config["threads"]["thread"]["min_replies"])
|
||||
min_engagement = int(settings.config["threads"]["thread"].get("min_engagement", 0))
|
||||
|
||||
with launch_browser(headless=True) as browser:
|
||||
context = ensure_authenticated_context(browser)
|
||||
|
||||
if POST_ID:
|
||||
post_url = f"https://www.threads.net/t/{POST_ID}"
|
||||
post = {"url": post_url, "post_id": POST_ID, "text": "", "body": ""}
|
||||
replies = _scrape_post_replies(context, post_url)
|
||||
content = _build_content_object(post, replies)
|
||||
if content["comments"] or content.get("thread_post"):
|
||||
return content
|
||||
raise RuntimeError(
|
||||
f"No replies found for post {POST_ID}. "
|
||||
f"Minimum required: {min_replies}."
|
||||
)
|
||||
|
||||
# Scrape from multiple sources: main feed + trending search queries
|
||||
posts = _scrape_feed_posts(context)
|
||||
# Also search for popular topics to find high-engagement content
|
||||
trending_queries = settings.config["threads"]["thread"].get(
|
||||
"search_queries", "news,politics,trending"
|
||||
)
|
||||
for query in trending_queries.split(","):
|
||||
query = query.strip()
|
||||
if query:
|
||||
try:
|
||||
search_posts = _scrape_search_page(context, query)
|
||||
# Merge avoiding duplicates
|
||||
existing_ids = {p["post_id"] for p in posts}
|
||||
for sp in search_posts:
|
||||
if sp["post_id"] not in existing_ids:
|
||||
posts.append(sp)
|
||||
except Exception as e:
|
||||
print_substep(f"Search query failed: {e}", "yellow")
|
||||
|
||||
if not posts:
|
||||
raise RuntimeError("No posts found in feed. Try again later.")
|
||||
|
||||
candidates = _filter_candidates(posts)
|
||||
if not candidates:
|
||||
raise RuntimeError(
|
||||
f"No eligible posts in feed after filtering. "
|
||||
f"Try lowering min_engagement (currently {min_engagement:,}) "
|
||||
f"or min_replies (currently {min_replies})."
|
||||
)
|
||||
|
||||
for i, candidate in enumerate(candidates):
|
||||
eng = candidate.get("_total_engagement", 0)
|
||||
print_substep(
|
||||
f"Trying #{i + 1}: ♥{candidate['likes']:,} "
|
||||
f"💬{candidate['replies_shown']} "
|
||||
f"'{candidate['body'][:60]}...'",
|
||||
style="dim",
|
||||
)
|
||||
emit_scraper_event("visiting_post", {
|
||||
"post_id": candidate["post_id"],
|
||||
"url": candidate["url"],
|
||||
"engagement": eng,
|
||||
"likes": candidate.get("likes", 0),
|
||||
"body": candidate.get("body", "")[:60],
|
||||
"attempt": i + 1,
|
||||
})
|
||||
try:
|
||||
replies = _scrape_post_replies(context, candidate["url"])
|
||||
emit_scraper_event("replies_found", {
|
||||
"post_id": candidate["post_id"],
|
||||
"count": len(replies),
|
||||
"min_required": min_replies,
|
||||
})
|
||||
if len(replies) >= min_replies:
|
||||
if not candidate.get("body") or len(candidate.get("body", "")) < 50:
|
||||
full_text = _scrape_main_post_text(context, candidate["url"])
|
||||
if full_text:
|
||||
candidate["body"] = full_text
|
||||
content = _build_content_object(candidate, replies)
|
||||
title_preview = content["thread_title"][:60]
|
||||
print_substep(
|
||||
f"Selected: '{title_preview}...' "
|
||||
f"♥{candidate['likes']:,} 💬{len(content['comments'])} replies",
|
||||
style="bold green",
|
||||
)
|
||||
emit_scraper_event("post_selected", {
|
||||
"title": content["thread_title"][:80],
|
||||
"post_id": candidate["post_id"],
|
||||
"likes": candidate["likes"],
|
||||
"replies_count": len(content["comments"]),
|
||||
"url": candidate["url"],
|
||||
})
|
||||
return content
|
||||
print_substep(
|
||||
f" Only {len(replies)} replies (need {min_replies}). Trying next...",
|
||||
style="yellow",
|
||||
)
|
||||
except Exception as e:
|
||||
print_substep(f" Failed: {e}. Trying next...", style="yellow")
|
||||
continue
|
||||
|
||||
raise RuntimeError(
|
||||
f"No eligible posts with {min_replies}+ replies found "
|
||||
f"after trying {len(candidates)} candidates."
|
||||
)
|
||||
@ -0,0 +1,185 @@
|
||||
"""Captures screenshots of Threads posts via the configured browser backend."""
|
||||
|
||||
import re
|
||||
from pathlib import Path
|
||||
from typing import Final
|
||||
|
||||
from playwright.sync_api import Page, ViewportSize
|
||||
|
||||
from platforms.threads.auth import ensure_authenticated_context
|
||||
from utils import settings
|
||||
from utils.browser_backend import launch_browser
|
||||
from utils.console import print_step, print_substep
|
||||
|
||||
|
||||
def _dismiss_popups(page: Page) -> None:
|
||||
"""Dismiss common Threads/Instagram overlays that block screenshots."""
|
||||
dismissals = [
|
||||
("[role='button']", "Not now"),
|
||||
("button", "Allow all cookies"),
|
||||
("[aria-label='Close']", None),
|
||||
("div[role='dialog'] button", "Not Now"),
|
||||
]
|
||||
for selector, text in dismissals:
|
||||
try:
|
||||
if text:
|
||||
el = page.locator(selector, has_text=text).first
|
||||
else:
|
||||
el = page.locator(selector).first
|
||||
if el.count() and el.is_visible():
|
||||
el.click()
|
||||
page.wait_for_timeout(500)
|
||||
print_substep(f"Dismissed popup: {selector}", style="dim")
|
||||
except Exception:
|
||||
pass
|
||||
|
||||
|
||||
def get_screenshots_of_threads_posts(content_object: dict, screenshot_num: int) -> None:
|
||||
"""
|
||||
Downloads screenshots of Threads posts via Playwright.
|
||||
|
||||
Args:
|
||||
content_object: Standard content dict from platforms/threads/fetcher.py
|
||||
screenshot_num: Number of reply screenshots to capture
|
||||
"""
|
||||
W: Final[int] = int(settings.config["settings"]["resolution_w"])
|
||||
H: Final[int] = int(settings.config["settings"]["resolution_h"])
|
||||
storymode: Final[bool] = settings.config["settings"]["storymode"]
|
||||
|
||||
print_step("Downloading screenshots of Threads posts...")
|
||||
|
||||
thread_id = re.sub(r"[^\w\s-]", "", content_object["thread_id"])
|
||||
Path(f"assets/temp/{thread_id}/png").mkdir(parents=True, exist_ok=True)
|
||||
|
||||
theme = settings.config["settings"]["theme"]
|
||||
|
||||
# Device scale factor (higher resolution screenshots)
|
||||
dsf = (W // 600) + 1
|
||||
|
||||
with launch_browser(headless=True) as browser:
|
||||
print_substep("Launching headless browser...")
|
||||
context = ensure_authenticated_context(
|
||||
browser,
|
||||
color_scheme="dark" if theme == "dark" else "light",
|
||||
viewport=ViewportSize(width=W, height=H),
|
||||
device_scale_factor=dsf,
|
||||
)
|
||||
|
||||
# Screenshot the main post
|
||||
page = context.new_page()
|
||||
page.goto(content_object["thread_url"], timeout=0)
|
||||
page.wait_for_load_state("networkidle")
|
||||
page.wait_for_timeout(3000)
|
||||
|
||||
_dismiss_popups(page)
|
||||
|
||||
if page.locator('div[role="dialog"] button:has-text("Log in")').first.is_visible():
|
||||
raise RuntimeError(
|
||||
"Threads login overlay is blocking the page — saved cookies are stale. "
|
||||
"Delete video_creation/data/cookie-threads.json and re-run."
|
||||
)
|
||||
|
||||
postcontentpath = f"assets/temp/{thread_id}/png/title.png"
|
||||
try:
|
||||
# Threads.net uses div-based cards, not <article> elements.
|
||||
# Find the first post link and screenshot its parent card.
|
||||
post_link = page.locator('a[href*="/post/"]').first
|
||||
if post_link.count() and post_link.is_visible():
|
||||
# Screenshot the card container, or fall back to the link's parent
|
||||
card = post_link.locator('xpath=ancestor::div[contains(@class, "x1a2a7pz")][1]')
|
||||
if card.count():
|
||||
post_locator = card.first
|
||||
else:
|
||||
post_locator = post_link
|
||||
else:
|
||||
# Fallback: try article (older Threads layout) or full page
|
||||
post_locator = page.locator("article").first
|
||||
if not post_locator.count() or not post_locator.is_visible():
|
||||
post_locator = page.locator("body")
|
||||
|
||||
if settings.config["settings"].get("zoom", 1) != 1:
|
||||
zoom = settings.config["settings"]["zoom"]
|
||||
page.evaluate(f"document.body.style.zoom={zoom}")
|
||||
location = post_locator.bounding_box()
|
||||
if location:
|
||||
for k in location:
|
||||
location[k] = float("{:.2f}".format(location[k] * zoom))
|
||||
page.screenshot(clip=location, path=postcontentpath)
|
||||
else:
|
||||
post_locator.screenshot(path=postcontentpath)
|
||||
else:
|
||||
post_locator.screenshot(path=postcontentpath)
|
||||
|
||||
print_substep("Main post screenshot captured.", style="bold green")
|
||||
except Exception as e:
|
||||
print_substep(f"Failed to screenshot main post: {e}", style="red")
|
||||
raise
|
||||
|
||||
# Screenshots of replies — capture all from the main post page (single navigation)
|
||||
if not storymode:
|
||||
num_replies = min(screenshot_num, len(content_object["comments"]))
|
||||
if num_replies > 0:
|
||||
# Scroll to load all replies inline on the post page
|
||||
print_substep("Loading replies on post page...", style="dim")
|
||||
_dismiss_popups(page)
|
||||
if page.locator('div[role="dialog"] button:has-text("Log in")').first.is_visible():
|
||||
raise RuntimeError(
|
||||
"Threads login overlay is blocking the page — saved cookies are stale. "
|
||||
"Delete video_creation/data/cookie-threads.json and re-run."
|
||||
)
|
||||
last_count = 0
|
||||
stable_count = 0
|
||||
for _ in range(20):
|
||||
page.evaluate("window.scrollBy(0, document.body.scrollHeight)")
|
||||
page.wait_for_timeout(1000)
|
||||
current = page.locator('a[href*="/post/"]').count()
|
||||
if current == last_count:
|
||||
stable_count += 1
|
||||
if stable_count >= 3:
|
||||
break
|
||||
else:
|
||||
stable_count = 0
|
||||
last_count = current
|
||||
|
||||
for idx in range(num_replies):
|
||||
comment = content_object["comments"][idx]
|
||||
try:
|
||||
reply_id = comment["comment_id"]
|
||||
reply_link = page.locator(f'a[href*="/{reply_id}"]').first
|
||||
if reply_link.count() and reply_link.is_visible():
|
||||
card = reply_link.locator('xpath=ancestor::div[contains(@class, "x1a2a7pz")][1]')
|
||||
reply_locator = card.first if card.count() else reply_link
|
||||
# Scroll element into view so bounding_box works
|
||||
reply_locator.scroll_into_view_if_needed()
|
||||
page.wait_for_timeout(300)
|
||||
else:
|
||||
print_substep(f"Reply {idx} not found on post page. Skipping...", style="yellow")
|
||||
continue
|
||||
|
||||
if settings.config["settings"].get("zoom", 1) != 1:
|
||||
zoom = settings.config["settings"]["zoom"]
|
||||
page.evaluate(f"document.body.style.zoom={zoom}")
|
||||
location = reply_locator.bounding_box()
|
||||
if location:
|
||||
for k in location:
|
||||
location[k] = float("{:.2f}".format(location[k] * zoom))
|
||||
page.screenshot(
|
||||
clip=location,
|
||||
path=f"assets/temp/{thread_id}/png/comment_{idx}.png",
|
||||
)
|
||||
else:
|
||||
reply_locator.screenshot(
|
||||
path=f"assets/temp/{thread_id}/png/comment_{idx}.png"
|
||||
)
|
||||
else:
|
||||
reply_locator.screenshot(
|
||||
path=f"assets/temp/{thread_id}/png/comment_{idx}.png"
|
||||
)
|
||||
|
||||
except Exception as e:
|
||||
print_substep(f"Error capturing reply {idx}: {e}. Skipping...", style="yellow")
|
||||
continue
|
||||
|
||||
print_substep(f"Reply screenshots captured ({num_replies} total).", style="bold green")
|
||||
|
||||
print_substep("Threads screenshots downloaded successfully.", style="bold green")
|
||||
@ -1,21 +1,26 @@
|
||||
boto3==1.36.8
|
||||
botocore==1.36.8
|
||||
boto3==1.43.3
|
||||
botocore==1.43.3
|
||||
gTTS==2.5.4
|
||||
moviepy==2.2.1
|
||||
playwright==1.49.1
|
||||
playwright==1.59.0
|
||||
cloakbrowser==0.3.28
|
||||
pyotp==2.9.0
|
||||
praw==7.8.1
|
||||
requests==2.32.3
|
||||
rich==13.9.4
|
||||
requests==2.33.1
|
||||
rich==15.0.0
|
||||
toml==0.10.2
|
||||
translators==5.9.9
|
||||
pyttsx3==2.98
|
||||
tomlkit==0.13.2
|
||||
Flask==3.1.1
|
||||
clean-text==0.6.0
|
||||
pyttsx3==2.99
|
||||
tomlkit==0.14.0
|
||||
Flask==3.1.3
|
||||
clean-text==0.7.1
|
||||
unidecode==1.4.0
|
||||
spacy==3.8.7
|
||||
torch==2.7.0
|
||||
transformers==4.52.4
|
||||
ffmpeg-python==0.2.0
|
||||
elevenlabs==1.57.0
|
||||
yt-dlp==2025.10.22
|
||||
torch==2.11.0
|
||||
transformers==4.57.6
|
||||
supertonic==1.3.1
|
||||
spacy==3.8.13
|
||||
av>=14.0
|
||||
elevenlabs==2.44.0
|
||||
yt-dlp==2026.3.17
|
||||
google-auth-oauthlib==1.2.1
|
||||
google-api-python-client==2.159.0
|
||||
|
||||
@ -1,2 +1,3 @@
|
||||
#!/bin/sh
|
||||
docker run -v $(pwd)/out/:/app/assets -v $(pwd)/.env:/app/.env -it rvmt
|
||||
set -eu
|
||||
docker compose run --rm cli "$@"
|
||||
|
||||
@ -0,0 +1,54 @@
|
||||
import sys
|
||||
from types import SimpleNamespace
|
||||
|
||||
from utils import settings
|
||||
|
||||
|
||||
class FakeBrowser:
|
||||
def __init__(self):
|
||||
self.closed = False
|
||||
|
||||
def close(self):
|
||||
self.closed = True
|
||||
|
||||
|
||||
def test_launch_browser_defaults_to_playwright(monkeypatch):
|
||||
from utils import browser_backend
|
||||
|
||||
fake_browser = FakeBrowser()
|
||||
|
||||
class FakePlaywrightContext:
|
||||
def __enter__(self):
|
||||
return SimpleNamespace(
|
||||
chromium=SimpleNamespace(launch=lambda headless: fake_browser)
|
||||
)
|
||||
|
||||
def __exit__(self, exc_type, exc, traceback):
|
||||
return False
|
||||
|
||||
settings.config = {"settings": {}}
|
||||
monkeypatch.setattr(browser_backend, "sync_playwright", lambda: FakePlaywrightContext())
|
||||
|
||||
with browser_backend.launch_browser() as browser:
|
||||
assert browser is fake_browser
|
||||
|
||||
assert fake_browser.closed is True
|
||||
|
||||
|
||||
def test_launch_browser_uses_cloakbrowser_when_configured(monkeypatch):
|
||||
from utils import browser_backend
|
||||
|
||||
fake_browser = FakeBrowser()
|
||||
launch_calls = []
|
||||
fake_module = SimpleNamespace(
|
||||
launch=lambda **kwargs: launch_calls.append(kwargs) or fake_browser
|
||||
)
|
||||
|
||||
settings.config = {"settings": {"browser": {"backend": "cloakbrowser"}}}
|
||||
monkeypatch.setitem(sys.modules, "cloakbrowser", fake_module)
|
||||
|
||||
with browser_backend.launch_browser(headless=False) as browser:
|
||||
assert browser is fake_browser
|
||||
|
||||
assert launch_calls == [{"headless": False, "humanize": True}]
|
||||
assert fake_browser.closed is True
|
||||
@ -0,0 +1,58 @@
|
||||
from unittest.mock import MagicMock, patch
|
||||
|
||||
import GUI
|
||||
from GUI import app
|
||||
|
||||
|
||||
def test_background_add_handles_missing_form_fields():
|
||||
app.testing = False
|
||||
|
||||
response = app.test_client().post("/background/add", data={})
|
||||
|
||||
assert response.status_code == 302
|
||||
assert response.headers["Location"].endswith("/backgrounds")
|
||||
|
||||
|
||||
def test_background_add_passes_empty_defaults_for_missing_optional_fields():
|
||||
app.testing = True
|
||||
|
||||
with patch("GUI.gui.add_background") as add_background:
|
||||
response = app.test_client().post(
|
||||
"/background/add",
|
||||
data={"youtube_uri": "https://www.youtube.com/watch?v=dQw4w9WgXcQ"},
|
||||
)
|
||||
|
||||
assert response.status_code == 302
|
||||
add_background.assert_called_once_with(
|
||||
"https://www.youtube.com/watch?v=dQw4w9WgXcQ",
|
||||
"",
|
||||
"",
|
||||
"",
|
||||
)
|
||||
|
||||
|
||||
def test_settings_get_uses_template_defaults_for_partial_config():
|
||||
app.testing = True
|
||||
fake_path = MagicMock()
|
||||
fake_path.read_text.return_value = ""
|
||||
|
||||
with patch("GUI.Path", MagicMock(return_value=fake_path)):
|
||||
response = app.test_client().get("/settings")
|
||||
|
||||
body = response.get_data(as_text=True)
|
||||
|
||||
assert response.status_code == 200
|
||||
assert '"settings.tts.voice_choice": "Supertonic"' in body
|
||||
assert 'name="settings.tts.supertonic_voice"' in body
|
||||
|
||||
|
||||
def test_public_demo_mode_blocks_mutating_routes(monkeypatch):
|
||||
app.testing = True
|
||||
monkeypatch.setattr(GUI, "PUBLIC_DEMO_MODE", True)
|
||||
client = app.test_client()
|
||||
|
||||
assert client.post("/background/add", data={}).status_code == 403
|
||||
assert client.post("/background/delete", data={}).status_code == 403
|
||||
assert client.post("/settings", data={}).status_code == 403
|
||||
assert client.post("/videos/delete", json={"ids": ["abc"]}).status_code == 403
|
||||
assert client.post("/create", json={}).status_code == 403
|
||||
@ -0,0 +1,131 @@
|
||||
import json
|
||||
import os
|
||||
import pytest
|
||||
from unittest.mock import patch, MagicMock
|
||||
from pathlib import Path
|
||||
from utils import gui_utils
|
||||
|
||||
@pytest.fixture
|
||||
def mock_background_json(tmp_path):
|
||||
bg_file = tmp_path / "background_videos.json"
|
||||
initial_data = {
|
||||
"__comment": "test",
|
||||
"minecraft": ["https://www.youtube.com/watch?v=n_Dv4JMiwK8", "parkour.mp4", "bbswitzer", "center"]
|
||||
}
|
||||
bg_file.write_text(json.dumps(initial_data))
|
||||
return bg_file
|
||||
|
||||
@pytest.fixture
|
||||
def mock_template_toml(tmp_path):
|
||||
template_file = tmp_path / ".config.template.toml"
|
||||
template_content = """
|
||||
[settings.background]
|
||||
background_video = { optional = true, default = "minecraft", options = ["minecraft"] }
|
||||
"""
|
||||
template_file.write_text(template_content)
|
||||
return template_file
|
||||
|
||||
@patch("utils.gui_utils.flash")
|
||||
def test_delete_background(mock_flash, mock_background_json, mock_template_toml):
|
||||
# We need to patch the paths used in gui_utils
|
||||
with patch("utils.gui_utils.open", MagicMock(side_effect=lambda path, *args, **kwargs: open(mock_background_json if "background_videos.json" in str(path) else path, *args, **kwargs))), \
|
||||
patch("utils.gui_utils.Path", MagicMock(side_effect=lambda path: Path(mock_template_toml) if ".config.template.toml" in str(path) else Path(path))):
|
||||
|
||||
gui_utils.delete_background("minecraft")
|
||||
|
||||
# Verify background_videos.json
|
||||
with open(mock_background_json, "r") as f:
|
||||
data = json.load(f)
|
||||
assert "minecraft" not in data
|
||||
|
||||
# Verify .config.template.toml
|
||||
import tomlkit
|
||||
template_data = tomlkit.loads(mock_template_toml.read_text())
|
||||
assert "minecraft" not in template_data["settings"]["background"]["background_video"]["options"]
|
||||
|
||||
mock_flash.assert_called_with('Successfully removed "minecraft" background!')
|
||||
|
||||
@patch("utils.gui_utils.flash")
|
||||
def test_add_background(mock_flash, mock_background_json, mock_template_toml):
|
||||
with patch("utils.gui_utils.open", MagicMock(side_effect=lambda path, *args, **kwargs: open(mock_background_json if "background_videos.json" in str(path) else path, *args, **kwargs))), \
|
||||
patch("utils.gui_utils.Path", MagicMock(side_effect=lambda path: Path(mock_template_toml) if ".config.template.toml" in str(path) else Path(path))):
|
||||
|
||||
# Test adding a new background
|
||||
gui_utils.add_background(
|
||||
youtube_uri="https://www.youtube.com/watch?v=dQw4w9WgXcQ",
|
||||
filename="test_new",
|
||||
citation="Rick",
|
||||
position="center"
|
||||
)
|
||||
|
||||
# Verify background_videos.json
|
||||
with open(mock_background_json, "r") as f:
|
||||
data = json.load(f)
|
||||
assert "test_new" in data
|
||||
assert data["test_new"][0] == "https://www.youtube.com/watch?v=dQw4w9WgXcQ"
|
||||
|
||||
# Verify .config.template.toml
|
||||
import tomlkit
|
||||
template_data = tomlkit.loads(mock_template_toml.read_text())
|
||||
assert "test_new" in template_data["settings"]["background"]["background_video"]["options"]
|
||||
|
||||
mock_flash.assert_called_with('Added "Rick-test_new.mp4" as a new background video!')
|
||||
|
||||
|
||||
@patch("utils.gui_utils.flash")
|
||||
def test_modify_settings_preserves_masked_secrets(mock_flash):
|
||||
config_load = {
|
||||
"reddit": {
|
||||
"creds": {
|
||||
"client_secret": "real-secret",
|
||||
"password": "real-password",
|
||||
}
|
||||
}
|
||||
}
|
||||
checks = {
|
||||
"reddit.creds.client_secret": {"optional": False, "type": "str"},
|
||||
"reddit.creds.password": {"optional": False, "type": "str"},
|
||||
}
|
||||
|
||||
result = gui_utils.modify_settings(
|
||||
{
|
||||
"reddit.creds.client_secret": "********",
|
||||
"reddit.creds.password": "changed-password",
|
||||
},
|
||||
config_load,
|
||||
checks,
|
||||
)
|
||||
|
||||
assert config_load["reddit"]["creds"]["client_secret"] == "real-secret"
|
||||
assert config_load["reddit"]["creds"]["password"] == "changed-password"
|
||||
assert result["reddit.creds.client_secret"] == "real-secret"
|
||||
|
||||
|
||||
@patch("utils.gui_utils.flash")
|
||||
def test_modify_settings_does_not_materialize_missing_default_values(mock_flash, tmp_path):
|
||||
config_load = {"settings": {"channel_name": "My Channel"}}
|
||||
checks = {
|
||||
"settings.channel_name": {"optional": True, "type": "str"},
|
||||
"settings.tts.voice_choice": {"optional": False, "type": "str", "default": "Supertonic"},
|
||||
"settings.tts.supertonic_voice": {"optional": True, "type": "str", "default": "M1"},
|
||||
"threads.creds.username": {"optional": True, "type": "str"},
|
||||
}
|
||||
config_path = tmp_path / "config.toml"
|
||||
|
||||
with patch("utils.gui_utils.Path", MagicMock(return_value=config_path)):
|
||||
result = gui_utils.modify_settings(
|
||||
{
|
||||
"settings.channel_name": "Updated Channel",
|
||||
"settings.tts.voice_choice": "Supertonic",
|
||||
"settings.tts.supertonic_voice": "M1",
|
||||
"threads.creds.username": "",
|
||||
},
|
||||
config_load,
|
||||
checks,
|
||||
)
|
||||
|
||||
assert config_load["settings"]["channel_name"] == "Updated Channel"
|
||||
assert "tts" not in config_load["settings"]
|
||||
assert "threads" not in config_load
|
||||
assert "settings.tts.voice_choice" not in result
|
||||
assert "threads.creds.username" not in result
|
||||
@ -0,0 +1,34 @@
|
||||
import os
|
||||
from unittest.mock import patch
|
||||
|
||||
import main
|
||||
|
||||
|
||||
def test_clear_screen_skips_clear_without_term(monkeypatch):
|
||||
monkeypatch.delenv("TERM", raising=False)
|
||||
monkeypatch.setattr(main, "name", "posix")
|
||||
|
||||
with patch("main.subprocess.run") as run:
|
||||
main.clear_screen()
|
||||
|
||||
run.assert_not_called()
|
||||
|
||||
|
||||
def test_clear_screen_runs_when_term_is_set(monkeypatch):
|
||||
monkeypatch.setenv("TERM", "xterm")
|
||||
monkeypatch.setattr(main, "name", "posix")
|
||||
|
||||
with patch("main.subprocess.run") as run:
|
||||
main.clear_screen()
|
||||
|
||||
run.assert_called_once_with(["clear"], shell=False)
|
||||
|
||||
|
||||
def test_clear_screen_runs_windows_command(monkeypatch):
|
||||
monkeypatch.delenv("TERM", raising=False)
|
||||
monkeypatch.setattr(main, "name", "nt")
|
||||
|
||||
with patch("main.subprocess.run") as run:
|
||||
main.clear_screen()
|
||||
|
||||
run.assert_called_once_with(["cls"], shell=True)
|
||||
@ -0,0 +1,43 @@
|
||||
from utils.settings import apply_template_defaults
|
||||
|
||||
|
||||
def test_apply_template_defaults_fills_missing_values_without_overwriting(tmp_path):
|
||||
template = tmp_path / "template.toml"
|
||||
template.write_text(
|
||||
"""
|
||||
[settings]
|
||||
platform = { optional = false, default = "reddit" }
|
||||
|
||||
[settings.tts]
|
||||
no_emojis = { optional = false, default = false, type = "bool" }
|
||||
|
||||
[threads.creds]
|
||||
access_token = { optional = false }
|
||||
username = { optional = true }
|
||||
|
||||
[threads.thread]
|
||||
min_replies = { optional = false, default = 5, type = "int" }
|
||||
search_queries = { optional = true, default = "news,politics,trending" }
|
||||
"""
|
||||
)
|
||||
config = {
|
||||
"settings": {"platform": "threads"},
|
||||
"threads": {"creds": {"username": "configured-user"}},
|
||||
}
|
||||
|
||||
result = apply_template_defaults(config, str(template))
|
||||
|
||||
assert result["settings"]["platform"] == "threads"
|
||||
assert result["settings"]["tts"]["no_emojis"] is False
|
||||
assert result["threads"]["thread"]["min_replies"] == 5
|
||||
assert result["threads"]["thread"]["search_queries"] == "news,politics,trending"
|
||||
assert result["threads"]["creds"]["username"] == "configured-user"
|
||||
assert "access_token" not in result["threads"]["creds"]
|
||||
|
||||
|
||||
def test_real_template_defaults_to_supertonic_tts():
|
||||
result = apply_template_defaults({"settings": {}})
|
||||
|
||||
assert result["settings"]["tts"]["voice_choice"] == "Supertonic"
|
||||
assert result["settings"]["tts"]["supertonic_voice"] == "M1"
|
||||
assert result["settings"]["tts"]["supertonic_lang"] == "na"
|
||||
@ -0,0 +1,25 @@
|
||||
from pathlib import Path
|
||||
|
||||
|
||||
def test_audio_settings_template_supports_supertonic_and_provider_visibility():
|
||||
template = Path("GUI/settings.html").read_text()
|
||||
|
||||
assert 'checks["settings.tts.voice_choice"]["options"]' in template
|
||||
assert 'data-tts-providers="Supertonic"' in template
|
||||
assert 'data-tts-providers="elevenlabs,OpenAI,tiktok"' in template
|
||||
assert 'const voiceChoiceSelect = document.getElementById(\'voiceChoiceSelect\');' in template
|
||||
assert 'applyTtsVisibility();' in template
|
||||
|
||||
|
||||
def test_threads_settings_template_supports_discovery_specific_credentials():
|
||||
template = Path("GUI/settings.html").read_text()
|
||||
|
||||
assert 'id="threadsDiscoveryMethodSelect"' in template
|
||||
assert 'data-discovery-methods="api"' in template
|
||||
assert 'data-discovery-methods="scrape"' in template
|
||||
assert 'data-preserve-hidden="true"' in template
|
||||
assert 'name="threads.creds.user_id"' in template
|
||||
assert 'Screenshots still reuse your saved Threads username/password.' in template
|
||||
assert "el.disabled = !isThreads || (!matches && !preserveHidden);" in template
|
||||
assert 'const threadsDiscoveryMethodSelect = document.getElementById(\'threadsDiscoveryMethodSelect\');' in template
|
||||
assert 'applyThreadsDiscoveryVisibility();' in template
|
||||
@ -0,0 +1,55 @@
|
||||
from pathlib import Path
|
||||
|
||||
from utils import settings
|
||||
|
||||
|
||||
class FakeSupertonicEngine:
|
||||
voice_style_names = ["M1", "F1"]
|
||||
|
||||
def __init__(self, auto_download=True):
|
||||
self.auto_download = auto_download
|
||||
|
||||
def get_voice_style(self, voice_name):
|
||||
return {"voice_name": voice_name}
|
||||
|
||||
def get_voice_style_from_path(self, path):
|
||||
return {"path": path}
|
||||
|
||||
def synthesize(self, text, **kwargs):
|
||||
return b"wav-data", 1.0
|
||||
|
||||
def save_audio(self, wav, path):
|
||||
Path(path).write_bytes(wav)
|
||||
|
||||
|
||||
def test_supertonic_tts_converts_wav_to_mp3(monkeypatch, tmp_path):
|
||||
import TTS.supertonic_tts as supertonic_module
|
||||
|
||||
run_calls = []
|
||||
|
||||
def fake_run(command, check):
|
||||
run_calls.append((command, check))
|
||||
Path(command[-1]).write_bytes(b"mp3-data")
|
||||
|
||||
settings.config = {
|
||||
"settings": {
|
||||
"tts": {
|
||||
"supertonic_voice": "M1",
|
||||
"supertonic_lang": "na",
|
||||
"supertonic_steps": 8,
|
||||
"supertonic_speed": 1.05,
|
||||
"supertonic_custom_voice_path": "",
|
||||
}
|
||||
}
|
||||
}
|
||||
monkeypatch.setattr(supertonic_module, "TTS", FakeSupertonicEngine)
|
||||
monkeypatch.setattr(supertonic_module.subprocess, "run", fake_run)
|
||||
|
||||
output_path = tmp_path / "voice.mp3"
|
||||
supertonic_module.SupertonicTTS().run("hello", str(output_path))
|
||||
|
||||
assert output_path.read_bytes() == b"mp3-data"
|
||||
command, check = run_calls[0]
|
||||
assert command[:5] == ["ffmpeg", "-y", "-hide_banner", "-loglevel", "error"]
|
||||
assert command[-1] == str(output_path)
|
||||
assert check is True
|
||||
@ -0,0 +1,10 @@
|
||||
from pathlib import Path
|
||||
|
||||
|
||||
def test_backgrounds_template_escapes_catalog_values_before_inner_html():
|
||||
template = Path("GUI/backgrounds.html").read_text(encoding="utf-8")
|
||||
|
||||
assert "function h(str)" in template
|
||||
assert "title=\"${h(key)}\"" in template
|
||||
assert "${h(key)}" in template
|
||||
assert "${h(value[2])}" in template
|
||||
@ -0,0 +1,59 @@
|
||||
from unittest.mock import MagicMock, patch
|
||||
|
||||
from playwright.sync_api import TimeoutError
|
||||
|
||||
from platforms.threads import auth
|
||||
from platforms.threads.auth import _is_logged_in_threads_url
|
||||
|
||||
|
||||
def test_logged_in_threads_url_accepts_current_threads_com_home():
|
||||
assert _is_logged_in_threads_url("https://www.threads.com/") is True
|
||||
assert _is_logged_in_threads_url("https://www.threads.com/?hl=en") is True
|
||||
|
||||
|
||||
def test_logged_in_threads_url_accepts_legacy_threads_net_home():
|
||||
assert _is_logged_in_threads_url("https://www.threads.net/") is True
|
||||
|
||||
|
||||
def test_logged_in_threads_url_rejects_login_page_and_other_hosts():
|
||||
assert _is_logged_in_threads_url("https://www.threads.com/login") is False
|
||||
assert _is_logged_in_threads_url("https://www.threads.net/login") is False
|
||||
assert _is_logged_in_threads_url("https://example.com/") is False
|
||||
|
||||
|
||||
def test_logged_in_threads_url_rejects_checkpoint_and_challenge_paths():
|
||||
assert _is_logged_in_threads_url("https://www.threads.com/challenge/") is False
|
||||
assert _is_logged_in_threads_url("https://www.threads.net/checkpoint/") is False
|
||||
assert _is_logged_in_threads_url("https://www.threads.com/consent/") is False
|
||||
|
||||
|
||||
def test_logged_in_threads_url_requires_https():
|
||||
assert _is_logged_in_threads_url("http://www.threads.com/") is False
|
||||
|
||||
|
||||
def test_login_to_threads_retries_with_enter_when_button_click_times_out(tmp_path):
|
||||
username_input = MagicMock()
|
||||
password_input = MagicMock()
|
||||
login_button = MagicMock()
|
||||
login_button.click.side_effect = TimeoutError("button click timed out")
|
||||
role_result = MagicMock()
|
||||
role_result.first = login_button
|
||||
|
||||
page = MagicMock()
|
||||
page.locator.side_effect = lambda selector: username_input if "username" in selector else password_input
|
||||
page.get_by_role.return_value = role_result
|
||||
|
||||
context = MagicMock()
|
||||
context.cookies.return_value = [{"name": "sessionid", "value": "ok"}]
|
||||
|
||||
cookie_path = tmp_path / "cookie-threads.json"
|
||||
|
||||
with patch.object(auth.settings, "config", {"threads": {"creds": {"username": "demo", "password": "secret"}}}), \
|
||||
patch.object(auth, "THREADS_COOKIE_FILE", str(cookie_path)):
|
||||
auth.login_to_threads(page, context)
|
||||
|
||||
username_input.fill.assert_called_once_with("demo")
|
||||
password_input.fill.assert_called_once_with("secret")
|
||||
password_input.press.assert_called_once_with("Enter")
|
||||
assert page.wait_for_url.call_count == 1
|
||||
assert cookie_path.exists()
|
||||
@ -0,0 +1,50 @@
|
||||
from pathlib import Path
|
||||
|
||||
from TTS.TikTok import TikTokTTSException
|
||||
from TTS.engine_wrapper import TTSEngine
|
||||
from utils import settings
|
||||
|
||||
|
||||
class FailingTikTok:
|
||||
max_chars = 5000
|
||||
|
||||
def run(self, text, filepath, random_voice=False):
|
||||
raise TikTokTTSException(1, "failed")
|
||||
|
||||
|
||||
class FakeGTTS:
|
||||
max_chars = 5000
|
||||
|
||||
def run(self, text, filepath):
|
||||
Path(filepath).write_text(text)
|
||||
|
||||
|
||||
class FakeAudioFileClip:
|
||||
duration = 1.0
|
||||
|
||||
def __init__(self, path):
|
||||
self.path = path
|
||||
|
||||
def close(self):
|
||||
pass
|
||||
|
||||
|
||||
def test_tiktok_fallback_uses_googletranslate_not_pyttsx(monkeypatch, tmp_path):
|
||||
import TTS.engine_wrapper as engine_wrapper
|
||||
|
||||
settings.config = {
|
||||
"settings": {
|
||||
"tts": {"voice_choice": "tiktok", "random_voice": False},
|
||||
}
|
||||
}
|
||||
reddit_object = {"thread_id": "abc", "thread_title": "title", "comments": []}
|
||||
monkeypatch.setattr(engine_wrapper, "GTTS", FakeGTTS)
|
||||
monkeypatch.setattr(engine_wrapper, "AudioFileClip", FakeAudioFileClip)
|
||||
|
||||
engine = TTSEngine(FailingTikTok, reddit_object, path=f"{tmp_path}/")
|
||||
Path(engine.path).mkdir(parents=True)
|
||||
|
||||
engine.call_tts("title", "hello")
|
||||
|
||||
assert settings.config["settings"]["tts"]["voice_choice"] == "googletranslate"
|
||||
assert Path(engine.path, "title.mp3").read_text() == "hello"
|
||||
@ -0,0 +1,47 @@
|
||||
"""Optional browser backend adapter for Playwright-compatible browsers."""
|
||||
|
||||
from contextlib import contextmanager
|
||||
from typing import Iterator
|
||||
|
||||
from playwright.sync_api import Browser, sync_playwright
|
||||
|
||||
from utils import settings
|
||||
|
||||
|
||||
def _configured_backend() -> str:
|
||||
config = settings.config if isinstance(settings.config, dict) else {}
|
||||
browser_config = config.get("settings", {}).get("browser", {})
|
||||
return str(browser_config.get("backend", "playwright")).casefold()
|
||||
|
||||
|
||||
@contextmanager
|
||||
def launch_browser(headless: bool = True) -> Iterator[Browser]:
|
||||
"""Launch the configured browser backend.
|
||||
|
||||
Defaults to stock Playwright. CloakBrowser is opt-in via
|
||||
settings.browser.backend = "cloakbrowser" and returns a Playwright-compatible
|
||||
Browser object, preserving browser.new_context(...) call sites.
|
||||
"""
|
||||
backend = _configured_backend()
|
||||
|
||||
if backend == "cloakbrowser":
|
||||
import cloakbrowser
|
||||
|
||||
browser = cloakbrowser.launch(headless=headless, humanize=True)
|
||||
try:
|
||||
yield browser
|
||||
finally:
|
||||
browser.close()
|
||||
return
|
||||
|
||||
if backend != "playwright":
|
||||
raise ValueError(
|
||||
f"Unsupported browser backend '{backend}'. Use 'playwright' or 'cloakbrowser'."
|
||||
)
|
||||
|
||||
with sync_playwright() as playwright:
|
||||
browser = playwright.chromium.launch(headless=headless)
|
||||
try:
|
||||
yield browser
|
||||
finally:
|
||||
browser.close()
|
||||
@ -0,0 +1,71 @@
|
||||
"""Container bootstrap helpers for first-run runtime state."""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
from pathlib import Path
|
||||
from typing import Any, Dict
|
||||
|
||||
import tomlkit
|
||||
|
||||
|
||||
ROOT = Path(__file__).resolve().parent.parent
|
||||
|
||||
|
||||
def _default_from_template(node: Dict[str, Any]) -> Dict[str, Any]:
|
||||
defaults: Dict[str, Any] = {}
|
||||
for key, value in node.items():
|
||||
if isinstance(value, dict) and "optional" in value:
|
||||
if "default" in value:
|
||||
defaults[key] = value["default"]
|
||||
else:
|
||||
value_type = value.get("type")
|
||||
if value_type == "bool":
|
||||
defaults[key] = False
|
||||
elif value_type in {"int", "float"}:
|
||||
defaults[key] = 0
|
||||
else:
|
||||
defaults[key] = ""
|
||||
elif isinstance(value, dict):
|
||||
defaults[key] = _default_from_template(value)
|
||||
return defaults
|
||||
|
||||
|
||||
def _ensure_json(path: Path, content: str) -> None:
|
||||
path.parent.mkdir(parents=True, exist_ok=True)
|
||||
if not path.exists():
|
||||
path.write_text(content, encoding="utf-8")
|
||||
|
||||
|
||||
def _ensure_config(path: Path) -> None:
|
||||
if path.exists():
|
||||
return
|
||||
|
||||
template_path = ROOT / "utils/.config.template.toml"
|
||||
template = tomlkit.loads(template_path.read_text(encoding="utf-8"))
|
||||
defaults = _default_from_template(template)
|
||||
path.write_text(tomlkit.dumps(defaults), encoding="utf-8")
|
||||
|
||||
|
||||
def ensure_runtime_state() -> None:
|
||||
"""Create runtime files and directories expected by the app."""
|
||||
for relative in (
|
||||
"assets/temp",
|
||||
"assets/backgrounds/audio",
|
||||
"assets/backgrounds/video",
|
||||
".cache",
|
||||
"results",
|
||||
"video_creation/data",
|
||||
):
|
||||
(ROOT / relative).mkdir(parents=True, exist_ok=True)
|
||||
|
||||
_ensure_config(ROOT / "config.toml")
|
||||
_ensure_json(ROOT / "video_creation/data/videos.json", "[]\n")
|
||||
_ensure_json(ROOT / "utils/backgrounds.json", "{}\n")
|
||||
|
||||
|
||||
def main() -> None:
|
||||
ensure_runtime_state()
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
main()
|
||||
@ -1,13 +1,14 @@
|
||||
from typing import Union
|
||||
from PIL.ImageFont import FreeTypeFont, ImageFont
|
||||
|
||||
|
||||
def getsize(font: ImageFont | FreeTypeFont, text: str):
|
||||
def getsize(font: Union[ImageFont, FreeTypeFont], text: str):
|
||||
left, top, right, bottom = font.getbbox(text)
|
||||
width = right - left
|
||||
height = bottom - top
|
||||
return width, height
|
||||
|
||||
|
||||
def getheight(font: ImageFont | FreeTypeFont, text: str):
|
||||
def getheight(font: Union[ImageFont, FreeTypeFont], text: str):
|
||||
_, height = getsize(font, text)
|
||||
return height
|
||||
|
||||
@ -0,0 +1,180 @@
|
||||
"""
|
||||
YouTube Uploader — OAuth2-authenticated upload to YouTube.
|
||||
|
||||
Imports the upload logic pattern from vendor/FullyAutomatedRedditVideoMakerBot/uploaders/youtubeUpload.py
|
||||
but is a standalone reimplementation that:
|
||||
- Reads config from the [youtube] section of config.toml
|
||||
- Lets the user point to their youtube_client_secret.json via config
|
||||
- Caches OAuth2 tokens to video_creation/data/YTtoken.json
|
||||
- Derives title, description, tags, privacy, category from config
|
||||
- Handles missing dependencies and missing secret files gracefully
|
||||
"""
|
||||
|
||||
import os
|
||||
import sys
|
||||
|
||||
from utils.console import print_markdown, print_step, print_substep
|
||||
|
||||
SCOPES = ["https://www.googleapis.com/auth/youtube.upload"]
|
||||
TOKEN_FILE = os.path.join("video_creation", "data", "YTtoken.json")
|
||||
|
||||
|
||||
def _get_authenticated_service(client_secret_path):
|
||||
"""
|
||||
Authenticate with YouTube via OAuth2.
|
||||
|
||||
Returns a googleapiclient.discovery.Resource (youtube v3) or None on failure.
|
||||
"""
|
||||
# Lazy imports so missing dependencies don't crash the pipeline
|
||||
try:
|
||||
from google.oauth2.credentials import Credentials
|
||||
from google_auth_oauthlib.flow import InstalledAppFlow
|
||||
from googleapiclient.discovery import build
|
||||
import google.auth.transport.requests
|
||||
except ImportError:
|
||||
print_substep(
|
||||
"YouTube upload requires google-auth-oauthlib and google-api-python-client.\n"
|
||||
"Install them with: pip install google-auth-oauthlib google-api-python-client",
|
||||
"bold red",
|
||||
)
|
||||
return None
|
||||
|
||||
# Validate client secret file exists
|
||||
if not client_secret_path or not os.path.isfile(client_secret_path):
|
||||
print_substep(
|
||||
f"YouTube client secret not found at: '{client_secret_path}'.\n"
|
||||
"Set youtube.client_secret_path in config.toml to the path of your "
|
||||
"youtube_client_secret.json file (downloaded from Google Cloud Console).",
|
||||
"bold red",
|
||||
)
|
||||
return None
|
||||
|
||||
credentials = None
|
||||
|
||||
# Load previously cached token if available
|
||||
if os.path.isfile(TOKEN_FILE):
|
||||
try:
|
||||
with open(TOKEN_FILE, "r") as f:
|
||||
credentials = Credentials.from_authorized_user_file(TOKEN_FILE, SCOPES)
|
||||
except Exception:
|
||||
credentials = None
|
||||
|
||||
# Refresh expired token or start fresh OAuth flow
|
||||
if not credentials or not credentials.valid:
|
||||
if credentials and credentials.expired and credentials.refresh_token:
|
||||
try:
|
||||
credentials.refresh(google.auth.transport.requests.Request())
|
||||
except Exception:
|
||||
credentials = None
|
||||
|
||||
if not credentials:
|
||||
try:
|
||||
print_substep(
|
||||
"Opening browser for YouTube OAuth2 authorization...",
|
||||
"blue",
|
||||
)
|
||||
flow = InstalledAppFlow.from_client_secrets_file(
|
||||
client_secret_path, SCOPES
|
||||
)
|
||||
credentials = flow.run_local_server(port=0)
|
||||
except Exception as e:
|
||||
print_substep(f"YouTube OAuth2 authentication failed: {e}", "bold red")
|
||||
return None
|
||||
|
||||
# Cache credentials for future runs
|
||||
os.makedirs(os.path.dirname(TOKEN_FILE), exist_ok=True)
|
||||
with open(TOKEN_FILE, "w") as f:
|
||||
f.write(credentials.to_json())
|
||||
print_substep("YouTube credentials cached to video_creation/data/YTtoken.json", "green")
|
||||
|
||||
return build("youtube", "v3", credentials=credentials)
|
||||
|
||||
|
||||
def upload_to_youtube(video_path, video_title, config):
|
||||
"""
|
||||
Upload a video to YouTube using settings from the [youtube] config section.
|
||||
|
||||
The function is safe to call even when youtube is disabled — it will
|
||||
return None immediately with a log message.
|
||||
|
||||
Args:
|
||||
video_path: Absolute or relative path to the .mp4 video file.
|
||||
video_title: Display title for the YouTube video (typically the
|
||||
thread title from the content object).
|
||||
config: Full application configuration dict (settings.config).
|
||||
|
||||
Returns:
|
||||
str — YouTube URL (https://youtu.be/VIDEO_ID) on success, or
|
||||
None if the upload is disabled, skipped, or failed.
|
||||
"""
|
||||
youtube_config = config.get("youtube", {})
|
||||
enabled = youtube_config.get("enabled", False)
|
||||
|
||||
if not enabled:
|
||||
print_substep(
|
||||
"YouTube upload skipped (youtube.enabled = false in config.toml).",
|
||||
"yellow",
|
||||
)
|
||||
return None
|
||||
|
||||
if not os.path.isfile(video_path):
|
||||
print_substep(f"Video file not found: {video_path}", "bold red")
|
||||
return None
|
||||
|
||||
client_secret_path = youtube_config.get("client_secret_path", "")
|
||||
|
||||
print_step("Uploading video to YouTube...")
|
||||
|
||||
youtube = _get_authenticated_service(client_secret_path)
|
||||
if youtube is None:
|
||||
return None
|
||||
|
||||
# Build upload metadata from config (with sensible defaults)
|
||||
tags_str = youtube_config.get("tags", "shorts, reddit")
|
||||
tags = [t.strip() for t in tags_str.split(",") if t.strip()]
|
||||
privacy = youtube_config.get("privacy", "public")
|
||||
category = youtube_config.get("category", "22")
|
||||
|
||||
description = youtube_config.get(
|
||||
"description",
|
||||
f"{video_title}\n\n#shorts #short #reddit",
|
||||
)
|
||||
|
||||
try:
|
||||
from googleapiclient.http import MediaFileUpload
|
||||
|
||||
body = {
|
||||
"snippet": {
|
||||
"title": video_title,
|
||||
"description": description,
|
||||
"tags": tags,
|
||||
"categoryId": category,
|
||||
},
|
||||
"status": {
|
||||
"privacyStatus": privacy,
|
||||
"madeForKids": False,
|
||||
},
|
||||
}
|
||||
|
||||
media = MediaFileUpload(video_path, chunksize=-1, resumable=True)
|
||||
request = youtube.videos().insert(
|
||||
part="snippet,status",
|
||||
body=body,
|
||||
media_body=media,
|
||||
)
|
||||
|
||||
response = None
|
||||
while response is None:
|
||||
status, response = request.next_chunk()
|
||||
if status:
|
||||
print_substep(
|
||||
f"Uploading... {int(status.progress() * 100)}% complete."
|
||||
)
|
||||
|
||||
video_url = f"https://youtu.be/{response['id']}"
|
||||
print_markdown(f"## Video uploaded successfully: {video_url}")
|
||||
return video_url
|
||||
|
||||
except Exception as e:
|
||||
print_substep(f"YouTube upload failed: {e}", "bold red")
|
||||
return None
|
||||
Loading…
Reference in new issue