commit
ade0e16510
@ -0,0 +1,141 @@
|
|||||||
|
import base64
|
||||||
|
from random import choice
|
||||||
|
from typing import Union, Optional
|
||||||
|
|
||||||
|
|
||||||
|
class BaseApiTTS:
|
||||||
|
max_chars: int
|
||||||
|
decode_base64: bool = False
|
||||||
|
|
||||||
|
@staticmethod
|
||||||
|
def text_len_sanitize(
|
||||||
|
text: str,
|
||||||
|
max_length: int,
|
||||||
|
) -> list:
|
||||||
|
"""
|
||||||
|
Splits text if it's too long to be a query
|
||||||
|
|
||||||
|
Args:
|
||||||
|
text: text to be sanitized
|
||||||
|
max_length: maximum length of the query
|
||||||
|
|
||||||
|
Returns:
|
||||||
|
Split text as a list
|
||||||
|
"""
|
||||||
|
# Split by comma or dot (else you can lose intonations), if there is non, split by groups of 299 chars
|
||||||
|
split_text = list(
|
||||||
|
map(lambda x: x.strip() if x.strip()[-1] != "." else x.strip()[:-1],
|
||||||
|
filter(lambda x: True if x else False, text.split(".")))
|
||||||
|
)
|
||||||
|
if split_text and all([chunk.__len__() < max_length for chunk in split_text]):
|
||||||
|
return split_text
|
||||||
|
|
||||||
|
split_text = list(
|
||||||
|
map(lambda x: x.strip() if x.strip()[-1] != "," else x.strip()[:-1],
|
||||||
|
filter(lambda x: True if x else False, text.split(","))
|
||||||
|
)
|
||||||
|
)
|
||||||
|
if split_text and all([chunk.__len__() < max_length for chunk in split_text]):
|
||||||
|
return split_text
|
||||||
|
|
||||||
|
return list(
|
||||||
|
map(
|
||||||
|
lambda x: x.strip() if x.strip()[-1] != "." or x.strip()[-1] != "," else x.strip()[:-1],
|
||||||
|
filter(
|
||||||
|
lambda x: True if x else False,
|
||||||
|
[text[i:i + max_length] for i in range(0, len(text), max_length)]
|
||||||
|
)
|
||||||
|
)
|
||||||
|
)
|
||||||
|
|
||||||
|
def write_file(
|
||||||
|
self,
|
||||||
|
output_text: str,
|
||||||
|
filepath: str,
|
||||||
|
) -> None:
|
||||||
|
"""
|
||||||
|
Writes and decodes TTS responses in files
|
||||||
|
|
||||||
|
Args:
|
||||||
|
output_text: text to be written
|
||||||
|
filepath: path/name of the file
|
||||||
|
"""
|
||||||
|
decoded_text = base64.b64decode(output_text) if self.decode_base64 else output_text
|
||||||
|
|
||||||
|
with open(filepath, "wb") as out:
|
||||||
|
out.write(decoded_text)
|
||||||
|
|
||||||
|
def run(
|
||||||
|
self,
|
||||||
|
text: str,
|
||||||
|
filepath: str,
|
||||||
|
) -> None:
|
||||||
|
"""
|
||||||
|
Calls for TTS api and writes audio file
|
||||||
|
|
||||||
|
Args:
|
||||||
|
text: text to be voice over
|
||||||
|
filepath: path/name of the file
|
||||||
|
|
||||||
|
Returns:
|
||||||
|
|
||||||
|
"""
|
||||||
|
output_text = ""
|
||||||
|
if len(text) > self.max_chars:
|
||||||
|
for part in self.text_len_sanitize(text, self.max_chars):
|
||||||
|
if part:
|
||||||
|
output_text += self.make_request(part)
|
||||||
|
else:
|
||||||
|
output_text = self.make_request(text)
|
||||||
|
self.write_file(output_text, filepath)
|
||||||
|
|
||||||
|
|
||||||
|
def get_random_voice(
|
||||||
|
voices: Union[list, dict],
|
||||||
|
key: Optional[str] = None,
|
||||||
|
) -> str:
|
||||||
|
"""
|
||||||
|
Return random voice from list or dict
|
||||||
|
|
||||||
|
Args:
|
||||||
|
voices: list or dict of voices
|
||||||
|
key: key of a dict if you are using one
|
||||||
|
|
||||||
|
Returns:
|
||||||
|
random voice as a str
|
||||||
|
"""
|
||||||
|
if isinstance(voices, list):
|
||||||
|
return choice(voices)
|
||||||
|
else:
|
||||||
|
return choice(voices[key] if key else list(voices.values())[0])
|
||||||
|
|
||||||
|
|
||||||
|
def audio_length(
|
||||||
|
path: str,
|
||||||
|
) -> Union[float, int]:
|
||||||
|
"""
|
||||||
|
Gets the length of the audio file
|
||||||
|
|
||||||
|
Args:
|
||||||
|
path: audio file path
|
||||||
|
|
||||||
|
Returns:
|
||||||
|
length in seconds as an int
|
||||||
|
"""
|
||||||
|
from moviepy.editor import AudioFileClip
|
||||||
|
|
||||||
|
try:
|
||||||
|
# please use something else here in the future
|
||||||
|
audio_clip = AudioFileClip(path)
|
||||||
|
audio_duration = audio_clip.duration
|
||||||
|
audio_clip.close()
|
||||||
|
return audio_duration
|
||||||
|
except Exception as e:
|
||||||
|
import logging
|
||||||
|
|
||||||
|
logger = logging.getLogger("tts_logger")
|
||||||
|
logger.setLevel(logging.ERROR)
|
||||||
|
handler = logging.FileHandler(".tts.log", mode="a+", encoding="utf-8")
|
||||||
|
logger.addHandler(handler)
|
||||||
|
logger.error("Error occurred in audio_length:", e)
|
||||||
|
return 0
|
@ -1,14 +0,0 @@
|
|||||||
[
|
|
||||||
{
|
|
||||||
"name": "USER",
|
|
||||||
"value": "eyJwcmVmcyI6eyJ0b3BDb250ZW50RGlzbWlzc2FsVGltZSI6MCwiZ2xvYmFsVGhlbWUiOiJSRURESVQiLCJuaWdodG1vZGUiOnRydWUsImNvbGxhcHNlZFRyYXlTZWN0aW9ucyI6eyJmYXZvcml0ZXMiOmZhbHNlLCJtdWx0aXMiOmZhbHNlLCJtb2RlcmF0aW5nIjpmYWxzZSwic3Vic2NyaXB0aW9ucyI6ZmFsc2UsInByb2ZpbGVzIjpmYWxzZX0sInRvcENvbnRlbnRUaW1lc0Rpc21pc3NlZCI6MH19",
|
|
||||||
"domain": ".reddit.com",
|
|
||||||
"path": "/"
|
|
||||||
},
|
|
||||||
{
|
|
||||||
"name": "eu_cookie",
|
|
||||||
"value": "{%22opted%22:true%2C%22nonessential%22:false}",
|
|
||||||
"domain": ".reddit.com",
|
|
||||||
"path": "/"
|
|
||||||
}
|
|
||||||
]
|
|
@ -1,8 +0,0 @@
|
|||||||
[
|
|
||||||
{
|
|
||||||
"name": "eu_cookie",
|
|
||||||
"value": "{%22opted%22:true%2C%22nonessential%22:false}",
|
|
||||||
"domain": ".reddit.com",
|
|
||||||
"path": "/"
|
|
||||||
}
|
|
||||||
]
|
|
@ -1,114 +0,0 @@
|
|||||||
import json
|
|
||||||
|
|
||||||
from pathlib import Path
|
|
||||||
from typing import Dict
|
|
||||||
from utils import settings
|
|
||||||
from playwright.async_api import async_playwright # pylint: disable=unused-import
|
|
||||||
|
|
||||||
# do not remove the above line
|
|
||||||
|
|
||||||
from playwright.sync_api import sync_playwright, ViewportSize
|
|
||||||
from rich.progress import track
|
|
||||||
import translators as ts
|
|
||||||
|
|
||||||
from utils.console import print_step, print_substep
|
|
||||||
|
|
||||||
storymode = False
|
|
||||||
|
|
||||||
|
|
||||||
def download_screenshots_of_reddit_posts(reddit_object: dict, screenshot_num: int):
|
|
||||||
"""Downloads screenshots of reddit posts as seen on the web. Downloads to assets/temp/png
|
|
||||||
|
|
||||||
Args:
|
|
||||||
reddit_object (Dict): Reddit object received from reddit/subreddit.py
|
|
||||||
screenshot_num (int): Number of screenshots to download
|
|
||||||
"""
|
|
||||||
print_step("Downloading screenshots of reddit posts...")
|
|
||||||
|
|
||||||
# ! Make sure the reddit screenshots folder exists
|
|
||||||
Path("assets/temp/png").mkdir(parents=True, exist_ok=True)
|
|
||||||
|
|
||||||
with sync_playwright() as p:
|
|
||||||
print_substep("Launching Headless Browser...")
|
|
||||||
|
|
||||||
browser = p.chromium.launch()
|
|
||||||
context = browser.new_context()
|
|
||||||
|
|
||||||
if settings.config["settings"]["theme"] == "dark":
|
|
||||||
cookie_file = open("./video_creation/data/cookie-dark-mode.json", encoding="utf-8")
|
|
||||||
else:
|
|
||||||
cookie_file = open("./video_creation/data/cookie-light-mode.json", encoding="utf-8")
|
|
||||||
cookies = json.load(cookie_file)
|
|
||||||
context.add_cookies(cookies) # load preference cookies
|
|
||||||
# Get the thread screenshot
|
|
||||||
page = context.new_page()
|
|
||||||
page.goto(reddit_object["thread_url"], timeout=0)
|
|
||||||
page.set_viewport_size(ViewportSize(width=1920, height=1080))
|
|
||||||
if page.locator('[data-testid="content-gate"]').is_visible():
|
|
||||||
# This means the post is NSFW and requires to click the proceed button.
|
|
||||||
|
|
||||||
print_substep("Post is NSFW. You are spicy...")
|
|
||||||
page.locator('[data-testid="content-gate"] button').click()
|
|
||||||
page.wait_for_load_state() # Wait for page to fully load
|
|
||||||
|
|
||||||
if page.locator('[data-click-id="text"] button').is_visible():
|
|
||||||
page.locator(
|
|
||||||
'[data-click-id="text"] button'
|
|
||||||
).click() # Remove "Click to see nsfw" Button in Screenshot
|
|
||||||
|
|
||||||
# translate code
|
|
||||||
|
|
||||||
if settings.config["reddit"]["thread"]["post_lang"]:
|
|
||||||
print_substep("Translating post...")
|
|
||||||
texts_in_tl = ts.google(
|
|
||||||
reddit_object["thread_title"],
|
|
||||||
to_language=settings.config["reddit"]["thread"]["post_lang"],
|
|
||||||
)
|
|
||||||
|
|
||||||
page.evaluate(
|
|
||||||
"tl_content => document.querySelector('[data-test-id=\"post-content\"] > div:nth-child(3) > div > div').textContent = tl_content",
|
|
||||||
texts_in_tl,
|
|
||||||
)
|
|
||||||
else:
|
|
||||||
print_substep("Skipping translation...")
|
|
||||||
|
|
||||||
page.locator('[data-test-id="post-content"]').screenshot(path="assets/temp/png/title.png")
|
|
||||||
|
|
||||||
if storymode:
|
|
||||||
page.locator('[data-click-id="text"]').screenshot(
|
|
||||||
path="assets/temp/png/story_content.png"
|
|
||||||
)
|
|
||||||
else:
|
|
||||||
for idx, comment in enumerate(
|
|
||||||
track(reddit_object["comments"], "Downloading screenshots...")
|
|
||||||
):
|
|
||||||
# Stop if we have reached the screenshot_num
|
|
||||||
if idx >= screenshot_num:
|
|
||||||
break
|
|
||||||
|
|
||||||
if page.locator('[data-testid="content-gate"]').is_visible():
|
|
||||||
page.locator('[data-testid="content-gate"] button').click()
|
|
||||||
|
|
||||||
page.goto(f'https://reddit.com{comment["comment_url"]}', timeout=0)
|
|
||||||
|
|
||||||
# translate code
|
|
||||||
|
|
||||||
if settings.config["reddit"]["thread"]["post_lang"]:
|
|
||||||
comment_tl = ts.google(
|
|
||||||
comment["comment_body"],
|
|
||||||
to_language=settings.config["reddit"]["thread"]["post_lang"],
|
|
||||||
)
|
|
||||||
page.evaluate(
|
|
||||||
'([tl_content, tl_id]) => document.querySelector(`#t1_${tl_id} > div:nth-child(2) > div > div[data-testid="comment"] > div`).textContent = tl_content',
|
|
||||||
[comment_tl, comment["comment_id"]],
|
|
||||||
)
|
|
||||||
try:
|
|
||||||
page.locator(f"#t1_{comment['comment_id']}").screenshot(
|
|
||||||
path=f"assets/temp/png/comment_{idx}.png"
|
|
||||||
)
|
|
||||||
except TimeoutError:
|
|
||||||
del reddit_object["comments"]
|
|
||||||
screenshot_num += 1
|
|
||||||
print("TimeoutError: Skipping screenshot...")
|
|
||||||
continue
|
|
||||||
print_substep("Screenshots downloaded Successfully.", style="bold green")
|
|
@ -0,0 +1,84 @@
|
|||||||
|
from attr import attrs, attrib
|
||||||
|
from typing import TypeVar, Optional, Callable, Union
|
||||||
|
|
||||||
|
|
||||||
|
_function = TypeVar("_function", bound=Callable[..., object])
|
||||||
|
_exceptions = TypeVar("_exceptions", bound=Optional[Union[type, tuple, list]])
|
||||||
|
|
||||||
|
default_exception = None
|
||||||
|
|
||||||
|
|
||||||
|
@attrs
|
||||||
|
class ExceptionDecorator:
|
||||||
|
"""
|
||||||
|
Decorator for catching exceptions and writing logs
|
||||||
|
"""
|
||||||
|
exception: Optional[_exceptions] = attrib(default=None)
|
||||||
|
|
||||||
|
def __attrs_post_init__(self):
|
||||||
|
if not self.exception:
|
||||||
|
self.exception = default_exception
|
||||||
|
|
||||||
|
def __call__(
|
||||||
|
self,
|
||||||
|
func: _function,
|
||||||
|
):
|
||||||
|
async def wrapper(*args, **kwargs):
|
||||||
|
try:
|
||||||
|
obj_to_return = await func(*args, **kwargs)
|
||||||
|
return obj_to_return
|
||||||
|
except Exception as caughtException:
|
||||||
|
import logging
|
||||||
|
|
||||||
|
logger = logging.getLogger("webdriver_log")
|
||||||
|
logger.setLevel(logging.ERROR)
|
||||||
|
handler = logging.FileHandler(".webdriver.log", mode="a+", encoding="utf-8")
|
||||||
|
logger.addHandler(handler)
|
||||||
|
|
||||||
|
if isinstance(self.exception, type):
|
||||||
|
if not type(caughtException) == self.exception:
|
||||||
|
logger.error(f"unexpected error - {caughtException}")
|
||||||
|
else:
|
||||||
|
if not type(caughtException) in self.exception:
|
||||||
|
logger.error(f"unexpected error - {caughtException}")
|
||||||
|
|
||||||
|
return wrapper
|
||||||
|
|
||||||
|
|
||||||
|
def catch_exception(
|
||||||
|
func: Optional[_function],
|
||||||
|
exception: Optional[_exceptions] = None,
|
||||||
|
) -> Union[object, _function]:
|
||||||
|
"""
|
||||||
|
Decorator for catching exceptions and writing logs
|
||||||
|
|
||||||
|
Args:
|
||||||
|
func: Function to be decorated
|
||||||
|
exception: Expected exception(s)
|
||||||
|
Returns:
|
||||||
|
Decorated function
|
||||||
|
"""
|
||||||
|
exceptor = ExceptionDecorator(exception)
|
||||||
|
if func:
|
||||||
|
exceptor = exceptor(func)
|
||||||
|
return exceptor
|
||||||
|
|
||||||
|
|
||||||
|
# Lots of tabs - lots of memory
|
||||||
|
# chunk needed to minimize memory required
|
||||||
|
def chunks(
|
||||||
|
array: list,
|
||||||
|
size: int,
|
||||||
|
):
|
||||||
|
"""
|
||||||
|
Yield successive n-sized chunks from list.
|
||||||
|
|
||||||
|
Args:
|
||||||
|
array: List to be chunked
|
||||||
|
size: size of a chunk
|
||||||
|
|
||||||
|
Returns:
|
||||||
|
Generator with chunked list
|
||||||
|
"""
|
||||||
|
for i in range(0, len(array), size):
|
||||||
|
yield array[i:i + size]
|
@ -0,0 +1,332 @@
|
|||||||
|
from asyncio import as_completed
|
||||||
|
from pathlib import Path
|
||||||
|
from typing import Dict, Optional
|
||||||
|
|
||||||
|
import translators as ts
|
||||||
|
from attr import attrs, attrib
|
||||||
|
from attr.validators import instance_of
|
||||||
|
from playwright.async_api import Browser, Playwright, Page, BrowserContext, ElementHandle
|
||||||
|
from playwright.async_api import async_playwright, TimeoutError
|
||||||
|
from rich.progress import track
|
||||||
|
|
||||||
|
from utils import settings
|
||||||
|
from utils.console import print_step, print_substep
|
||||||
|
|
||||||
|
import webdriver.common as common
|
||||||
|
|
||||||
|
common.default_exception = TimeoutError
|
||||||
|
|
||||||
|
|
||||||
|
@attrs
|
||||||
|
class Browser:
|
||||||
|
"""
|
||||||
|
Args:
|
||||||
|
default_Viewport (dict):Pyppeteer Browser default_Viewport options
|
||||||
|
browser (BrowserCls): Pyppeteer Browser instance
|
||||||
|
"""
|
||||||
|
default_Viewport: dict = attrib(
|
||||||
|
validator=instance_of(dict),
|
||||||
|
default={
|
||||||
|
# 9x21 to see long posts
|
||||||
|
"width": 500,
|
||||||
|
"height": 1200,
|
||||||
|
},
|
||||||
|
kw_only=True,
|
||||||
|
)
|
||||||
|
playwright: Playwright
|
||||||
|
browser: Browser
|
||||||
|
context: BrowserContext
|
||||||
|
|
||||||
|
async def get_browser(
|
||||||
|
self,
|
||||||
|
) -> None:
|
||||||
|
"""
|
||||||
|
Creates Playwright instance & browser
|
||||||
|
"""
|
||||||
|
self.playwright = await async_playwright().start()
|
||||||
|
self.browser = await self.playwright.chromium.launch()
|
||||||
|
self.context = await self.browser.new_context(viewport=self.default_Viewport)
|
||||||
|
|
||||||
|
async def close_browser(
|
||||||
|
self,
|
||||||
|
) -> None:
|
||||||
|
"""
|
||||||
|
Closes Playwright stuff
|
||||||
|
"""
|
||||||
|
await self.context.close()
|
||||||
|
await self.browser.close()
|
||||||
|
await self.playwright.stop()
|
||||||
|
|
||||||
|
|
||||||
|
class Flaky:
|
||||||
|
"""
|
||||||
|
All methods decorated with function catching default exceptions and writing logs
|
||||||
|
"""
|
||||||
|
|
||||||
|
@staticmethod
|
||||||
|
@common.catch_exception
|
||||||
|
async def find_element(
|
||||||
|
selector: str,
|
||||||
|
page_instance: Page,
|
||||||
|
options: Optional[dict] = None,
|
||||||
|
) -> ElementHandle:
|
||||||
|
return (
|
||||||
|
await page_instance.wait_for_selector(selector, **options)
|
||||||
|
if options
|
||||||
|
else await page_instance.wait_for_selector(selector)
|
||||||
|
)
|
||||||
|
|
||||||
|
@common.catch_exception
|
||||||
|
async def click(
|
||||||
|
self,
|
||||||
|
page_instance: Optional[Page] = None,
|
||||||
|
query: Optional[str] = None,
|
||||||
|
options: Optional[dict] = None,
|
||||||
|
*,
|
||||||
|
find_options: Optional[dict] = None,
|
||||||
|
element: Optional[ElementHandle] = None,
|
||||||
|
) -> None:
|
||||||
|
if element:
|
||||||
|
await element.click(**options) if options else await element.click()
|
||||||
|
else:
|
||||||
|
results = (
|
||||||
|
await self.find_element(query, page_instance, **find_options)
|
||||||
|
if find_options
|
||||||
|
else await self.find_element(query, page_instance)
|
||||||
|
)
|
||||||
|
await results.click(**options) if options else await results.click()
|
||||||
|
|
||||||
|
@common.catch_exception
|
||||||
|
async def screenshot(
|
||||||
|
self,
|
||||||
|
page_instance: Optional[Page] = None,
|
||||||
|
query: Optional[str] = None,
|
||||||
|
options: Optional[dict] = None,
|
||||||
|
*,
|
||||||
|
find_options: Optional[dict] = None,
|
||||||
|
element: Optional[ElementHandle] = None,
|
||||||
|
) -> None:
|
||||||
|
if element:
|
||||||
|
await element.screenshot(**options) if options else await element.screenshot()
|
||||||
|
else:
|
||||||
|
results = (
|
||||||
|
await self.find_element(query, page_instance, **find_options)
|
||||||
|
if find_options
|
||||||
|
else await self.find_element(query, page_instance)
|
||||||
|
)
|
||||||
|
await results.screenshot(**options) if options else await results.screenshot()
|
||||||
|
|
||||||
|
|
||||||
|
@attrs(auto_attribs=True)
|
||||||
|
class RedditScreenshot(Flaky, Browser):
|
||||||
|
"""
|
||||||
|
Args:
|
||||||
|
reddit_object (Dict): Reddit object received from reddit/subreddit.py
|
||||||
|
screenshot_idx (int): List with indexes of voiced comments
|
||||||
|
story_mode (bool): If submission is a story takes screenshot of the story
|
||||||
|
"""
|
||||||
|
reddit_object: dict
|
||||||
|
screenshot_idx: list
|
||||||
|
story_mode: Optional[bool] = attrib(
|
||||||
|
validator=instance_of(bool),
|
||||||
|
default=False,
|
||||||
|
kw_only=True
|
||||||
|
)
|
||||||
|
|
||||||
|
def __attrs_post_init__(
|
||||||
|
self
|
||||||
|
):
|
||||||
|
self.post_lang: Optional[bool] = settings.config["reddit"]["thread"]["post_lang"]
|
||||||
|
|
||||||
|
async def __dark_theme( # TODO isn't working
|
||||||
|
self,
|
||||||
|
page_instance: Page,
|
||||||
|
) -> None:
|
||||||
|
"""
|
||||||
|
Enables dark theme in Reddit
|
||||||
|
|
||||||
|
Args:
|
||||||
|
page_instance: Pyppeteer page instance with reddit page opened
|
||||||
|
"""
|
||||||
|
|
||||||
|
await self.click(
|
||||||
|
page_instance,
|
||||||
|
".header-user-dropdown",
|
||||||
|
)
|
||||||
|
|
||||||
|
# It's normal not to find it, sometimes there is none :shrug:
|
||||||
|
await self.click(
|
||||||
|
page_instance,
|
||||||
|
"button >> span:has-text('Settings')",
|
||||||
|
)
|
||||||
|
|
||||||
|
await self.click(
|
||||||
|
page_instance,
|
||||||
|
"button >> span:has-text('Dark Mode')",
|
||||||
|
)
|
||||||
|
|
||||||
|
# Closes settings
|
||||||
|
await self.click(
|
||||||
|
page_instance,
|
||||||
|
".header-user-dropdown"
|
||||||
|
)
|
||||||
|
|
||||||
|
async def __close_nsfw(
|
||||||
|
self,
|
||||||
|
page_instance: Page,
|
||||||
|
) -> None:
|
||||||
|
"""
|
||||||
|
Closes NSFW stuff
|
||||||
|
|
||||||
|
Args:
|
||||||
|
page_instance: Instance of main page
|
||||||
|
"""
|
||||||
|
|
||||||
|
print_substep("Post is NSFW. You are spicy...")
|
||||||
|
|
||||||
|
# Triggers indirectly reload
|
||||||
|
await self.click(
|
||||||
|
page_instance,
|
||||||
|
"button:has-text('Yes')",
|
||||||
|
{"timeout": 5000},
|
||||||
|
)
|
||||||
|
|
||||||
|
# Await indirect reload
|
||||||
|
await page_instance.wait_for_load_state()
|
||||||
|
|
||||||
|
await self.click(
|
||||||
|
page_instance,
|
||||||
|
"button:has-text('Click to see nsfw')",
|
||||||
|
{"timeout": 5000},
|
||||||
|
)
|
||||||
|
|
||||||
|
async def __collect_comment(
|
||||||
|
self,
|
||||||
|
comment_obj: dict,
|
||||||
|
filename_idx: int,
|
||||||
|
) -> None:
|
||||||
|
"""
|
||||||
|
Makes a screenshot of the comment
|
||||||
|
|
||||||
|
Args:
|
||||||
|
comment_obj: prew comment object
|
||||||
|
filename_idx: index for the filename
|
||||||
|
"""
|
||||||
|
comment_page = await self.context.new_page()
|
||||||
|
await comment_page.goto(f'https://reddit.com{comment_obj["comment_url"]}')
|
||||||
|
|
||||||
|
# Translates submission' comment
|
||||||
|
if self.post_lang:
|
||||||
|
comment_tl = ts.google(
|
||||||
|
comment_obj["comment_body"],
|
||||||
|
to_language=self.post_lang,
|
||||||
|
)
|
||||||
|
await comment_page.evaluate(
|
||||||
|
'([comment_id, comment_tl]) => document.querySelector(`#t1_${comment_id} > div:nth-child(2) > div > div[data-testid="comment"] > div`).textContent = comment_tl', # noqa
|
||||||
|
[comment_obj["comment_id"], comment_tl],
|
||||||
|
)
|
||||||
|
|
||||||
|
await self.screenshot(
|
||||||
|
comment_page,
|
||||||
|
f"id=t1_{comment_obj['comment_id']}",
|
||||||
|
{"path": f"assets/temp/png/comment_{filename_idx}.png"},
|
||||||
|
)
|
||||||
|
|
||||||
|
# WIP TODO test it
|
||||||
|
async def __collect_story(
|
||||||
|
self,
|
||||||
|
main_page: Page,
|
||||||
|
):
|
||||||
|
# Translates submission text
|
||||||
|
if self.post_lang:
|
||||||
|
story_tl = ts.google(
|
||||||
|
self.reddit_object["thread_post"],
|
||||||
|
to_language=self.post_lang,
|
||||||
|
)
|
||||||
|
split_story_tl = story_tl.split('\n')
|
||||||
|
|
||||||
|
await main_page.evaluate(
|
||||||
|
"(split_story_tl) => split_story_tl.map(function(element, i) { return [element, document.querySelectorAll('[data-test-id=\"post-content\"] > [data-click-id=\"text\"] > div > p')[i]]; }).forEach(mappedElement => mappedElement[1].textContent = mappedElement[0])", # noqa
|
||||||
|
split_story_tl,
|
||||||
|
)
|
||||||
|
|
||||||
|
await self.screenshot(
|
||||||
|
main_page,
|
||||||
|
"//div[@data-test-id='post-content']//div[@data-click-id='text']",
|
||||||
|
{"path": "assets/temp/png/story_content.png"},
|
||||||
|
)
|
||||||
|
|
||||||
|
async def download(
|
||||||
|
self,
|
||||||
|
):
|
||||||
|
"""
|
||||||
|
Downloads screenshots of reddit posts as seen on the web. Downloads to assets/temp/png
|
||||||
|
"""
|
||||||
|
print_step("Downloading screenshots of reddit posts...")
|
||||||
|
|
||||||
|
print_substep("Launching Headless Browser...")
|
||||||
|
await self.get_browser()
|
||||||
|
|
||||||
|
# ! Make sure the reddit screenshots folder exists
|
||||||
|
Path("assets/temp/png").mkdir(parents=True, exist_ok=True)
|
||||||
|
|
||||||
|
# Get the thread screenshot
|
||||||
|
reddit_main = await self.context.new_page()
|
||||||
|
await reddit_main.goto(self.reddit_object["thread_url"]) # noqa
|
||||||
|
|
||||||
|
if settings.config["settings"]["theme"] == "dark":
|
||||||
|
await self.__dark_theme(reddit_main)
|
||||||
|
|
||||||
|
if self.reddit_object["is_nsfw"]:
|
||||||
|
# This means the post is NSFW and requires to click the proceed button.
|
||||||
|
await self.__close_nsfw(reddit_main)
|
||||||
|
|
||||||
|
# Translates submission title
|
||||||
|
if self.post_lang:
|
||||||
|
print_substep("Translating post...")
|
||||||
|
texts_in_tl = ts.google(
|
||||||
|
self.reddit_object["thread_title"],
|
||||||
|
to_language=self.post_lang,
|
||||||
|
)
|
||||||
|
|
||||||
|
await reddit_main.evaluate(
|
||||||
|
f"(texts_in_tl) => document.querySelector('[data-test-id=\"post-content\"] > div:nth-child(3) > div > div').textContent = texts_in_tl", # noqa
|
||||||
|
texts_in_tl,
|
||||||
|
)
|
||||||
|
else:
|
||||||
|
print_substep("Skipping translation...")
|
||||||
|
|
||||||
|
# No sense to move it to common.py
|
||||||
|
async_tasks_primary = ( # noqa
|
||||||
|
[
|
||||||
|
self.__collect_comment(self.reddit_object["comments"][idx], idx) for idx in
|
||||||
|
self.screenshot_idx
|
||||||
|
]
|
||||||
|
if not self.story_mode
|
||||||
|
else [
|
||||||
|
self.__collect_story(reddit_main)
|
||||||
|
]
|
||||||
|
)
|
||||||
|
|
||||||
|
async_tasks_primary.append(
|
||||||
|
self.screenshot(
|
||||||
|
reddit_main,
|
||||||
|
f"id=t3_{self.reddit_object['thread_id']}",
|
||||||
|
{"path": "assets/temp/png/title.png"},
|
||||||
|
)
|
||||||
|
)
|
||||||
|
|
||||||
|
for idx, chunked_tasks in enumerate(
|
||||||
|
[chunk for chunk in common.chunks(async_tasks_primary, 10)],
|
||||||
|
start=1,
|
||||||
|
):
|
||||||
|
chunk_list = async_tasks_primary.__len__() // 10 + (1 if async_tasks_primary.__len__() % 10 != 0 else 0)
|
||||||
|
for task in track(
|
||||||
|
as_completed(chunked_tasks),
|
||||||
|
description=f"Downloading comments: Chunk {idx}/{chunk_list}",
|
||||||
|
total=chunked_tasks.__len__(),
|
||||||
|
):
|
||||||
|
await task
|
||||||
|
|
||||||
|
print_substep("Comments downloaded Successfully.", style="bold green")
|
||||||
|
await self.close_browser()
|
@ -0,0 +1,371 @@
|
|||||||
|
from asyncio import as_completed
|
||||||
|
|
||||||
|
from pyppeteer import launch
|
||||||
|
from pyppeteer.page import Page as PageCls
|
||||||
|
from pyppeteer.browser import Browser as BrowserCls
|
||||||
|
from pyppeteer.element_handle import ElementHandle as ElementHandleCls
|
||||||
|
from pyppeteer.errors import TimeoutError as BrowserTimeoutError
|
||||||
|
|
||||||
|
from pathlib import Path
|
||||||
|
from utils import settings
|
||||||
|
from utils.console import print_step, print_substep
|
||||||
|
from rich.progress import track
|
||||||
|
import translators as ts
|
||||||
|
|
||||||
|
from attr import attrs, attrib
|
||||||
|
from attr.validators import instance_of
|
||||||
|
from typing import Optional
|
||||||
|
|
||||||
|
import webdriver.common as common
|
||||||
|
|
||||||
|
common.default_exception = BrowserTimeoutError
|
||||||
|
|
||||||
|
|
||||||
|
@attrs
|
||||||
|
class Browser:
|
||||||
|
"""
|
||||||
|
Args:
|
||||||
|
default_Viewport (dict):Pyppeteer Browser default_Viewport options
|
||||||
|
browser (BrowserCls): Pyppeteer Browser instance
|
||||||
|
"""
|
||||||
|
default_Viewport: dict = attrib(
|
||||||
|
validator=instance_of(dict),
|
||||||
|
default={
|
||||||
|
# 9x21 to see long posts
|
||||||
|
"defaultViewport": {
|
||||||
|
"width": 500,
|
||||||
|
"height": 1200,
|
||||||
|
},
|
||||||
|
},
|
||||||
|
kw_only=True,
|
||||||
|
)
|
||||||
|
browser: BrowserCls
|
||||||
|
|
||||||
|
async def get_browser(
|
||||||
|
self,
|
||||||
|
) -> None:
|
||||||
|
"""
|
||||||
|
Creates Pyppeteer browser
|
||||||
|
"""
|
||||||
|
self.browser = await launch(self.default_Viewport)
|
||||||
|
|
||||||
|
async def close_browser(
|
||||||
|
self,
|
||||||
|
) -> None:
|
||||||
|
"""
|
||||||
|
Closes Pyppeteer browser
|
||||||
|
"""
|
||||||
|
await self.browser.close()
|
||||||
|
|
||||||
|
|
||||||
|
class Wait:
|
||||||
|
@staticmethod
|
||||||
|
@common.catch_exception
|
||||||
|
async def find_xpath(
|
||||||
|
page_instance: PageCls,
|
||||||
|
xpath: Optional[str] = None,
|
||||||
|
options: Optional[dict] = None,
|
||||||
|
) -> 'ElementHandleCls':
|
||||||
|
"""
|
||||||
|
Explicitly finds element on the page
|
||||||
|
|
||||||
|
Args:
|
||||||
|
page_instance: Pyppeteer page instance
|
||||||
|
xpath: xpath query
|
||||||
|
options: Pyppeteer waitForXPath parameters
|
||||||
|
|
||||||
|
Available options are:
|
||||||
|
|
||||||
|
* ``visible`` (bool): wait for element to be present in DOM and to be
|
||||||
|
visible, i.e. to not have ``display: none`` or ``visibility: hidden``
|
||||||
|
CSS properties. Defaults to ``False``.
|
||||||
|
* ``hidden`` (bool): wait for element to not be found in the DOM or to
|
||||||
|
be hidden, i.e. have ``display: none`` or ``visibility: hidden`` CSS
|
||||||
|
properties. Defaults to ``False``.
|
||||||
|
* ``timeout`` (int|float): maximum time to wait for in milliseconds.
|
||||||
|
Defaults to 30000 (30 seconds). Pass ``0`` to disable timeout.
|
||||||
|
Returns:
|
||||||
|
Pyppeteer element instance
|
||||||
|
"""
|
||||||
|
if options:
|
||||||
|
el = await page_instance.waitForXPath(xpath, options=options)
|
||||||
|
else:
|
||||||
|
el = await page_instance.waitForXPath(xpath)
|
||||||
|
return el
|
||||||
|
|
||||||
|
@common.catch_exception
|
||||||
|
async def click(
|
||||||
|
self,
|
||||||
|
page_instance: Optional[PageCls] = None,
|
||||||
|
xpath: Optional[str] = None,
|
||||||
|
options: Optional[dict] = None,
|
||||||
|
*,
|
||||||
|
find_options: Optional[dict] = None,
|
||||||
|
el: Optional[ElementHandleCls] = None,
|
||||||
|
) -> None:
|
||||||
|
"""
|
||||||
|
Clicks on the element
|
||||||
|
|
||||||
|
Args:
|
||||||
|
page_instance: Pyppeteer page instance
|
||||||
|
xpath: xpath query
|
||||||
|
find_options: Pyppeteer waitForXPath parameters
|
||||||
|
options: Pyppeteer click parameters
|
||||||
|
el: Pyppeteer element instance
|
||||||
|
"""
|
||||||
|
if not el:
|
||||||
|
el = await self.find_xpath(page_instance, xpath, find_options)
|
||||||
|
if options:
|
||||||
|
await el.click(options)
|
||||||
|
else:
|
||||||
|
await el.click()
|
||||||
|
|
||||||
|
@common.catch_exception
|
||||||
|
async def screenshot(
|
||||||
|
self,
|
||||||
|
page_instance: Optional[PageCls] = None,
|
||||||
|
xpath: Optional[str] = None,
|
||||||
|
options: Optional[dict] = None,
|
||||||
|
*,
|
||||||
|
find_options: Optional[dict] = None,
|
||||||
|
el: Optional[ElementHandleCls] = None,
|
||||||
|
) -> None:
|
||||||
|
"""
|
||||||
|
Makes a screenshot of the element
|
||||||
|
|
||||||
|
Args:
|
||||||
|
page_instance: Pyppeteer page instance
|
||||||
|
xpath: xpath query
|
||||||
|
options: Pyppeteer screenshot parameters
|
||||||
|
find_options: Pyppeteer waitForXPath parameters
|
||||||
|
el: Pyppeteer element instance
|
||||||
|
"""
|
||||||
|
if not el:
|
||||||
|
el = await self.find_xpath(page_instance, xpath, find_options)
|
||||||
|
if options:
|
||||||
|
await el.screenshot(options)
|
||||||
|
else:
|
||||||
|
await el.screenshot()
|
||||||
|
|
||||||
|
|
||||||
|
@attrs(auto_attribs=True)
|
||||||
|
class RedditScreenshot(Browser, Wait):
|
||||||
|
"""
|
||||||
|
Args:
|
||||||
|
reddit_object (Dict): Reddit object received from reddit/subreddit.py
|
||||||
|
screenshot_idx (int): List with indexes of voiced comments
|
||||||
|
story_mode (bool): If submission is a story takes screenshot of the story
|
||||||
|
"""
|
||||||
|
reddit_object: dict
|
||||||
|
screenshot_idx: list
|
||||||
|
story_mode: Optional[bool] = attrib(
|
||||||
|
validator=instance_of(bool),
|
||||||
|
default=False,
|
||||||
|
kw_only=True
|
||||||
|
)
|
||||||
|
|
||||||
|
def __attrs_post_init__(
|
||||||
|
self,
|
||||||
|
):
|
||||||
|
self.post_lang: Optional[bool] = settings.config["reddit"]["thread"]["post_lang"]
|
||||||
|
|
||||||
|
async def __dark_theme(
|
||||||
|
self,
|
||||||
|
page_instance: PageCls,
|
||||||
|
) -> None:
|
||||||
|
"""
|
||||||
|
Enables dark theme in Reddit
|
||||||
|
|
||||||
|
Args:
|
||||||
|
page_instance: Pyppeteer page instance with reddit page opened
|
||||||
|
"""
|
||||||
|
|
||||||
|
await self.click(
|
||||||
|
page_instance,
|
||||||
|
"//div[@class='header-user-dropdown']",
|
||||||
|
find_options={"timeout": 5000},
|
||||||
|
)
|
||||||
|
|
||||||
|
# It's normal not to find it, sometimes there is none :shrug:
|
||||||
|
await self.click(
|
||||||
|
page_instance,
|
||||||
|
"//span[text()='Settings']/ancestor::button[1]",
|
||||||
|
find_options={"timeout": 5000},
|
||||||
|
)
|
||||||
|
|
||||||
|
await self.click(
|
||||||
|
page_instance,
|
||||||
|
"//span[text()='Dark Mode']/ancestor::button[1]",
|
||||||
|
find_options={"timeout": 5000},
|
||||||
|
)
|
||||||
|
|
||||||
|
# Closes settings
|
||||||
|
await self.click(
|
||||||
|
page_instance,
|
||||||
|
"//div[@class='header-user-dropdown']",
|
||||||
|
find_options={"timeout": 5000},
|
||||||
|
)
|
||||||
|
|
||||||
|
async def __close_nsfw(
|
||||||
|
self,
|
||||||
|
page_instance: PageCls,
|
||||||
|
) -> None:
|
||||||
|
"""
|
||||||
|
Closes NSFW stuff
|
||||||
|
|
||||||
|
Args:
|
||||||
|
page_instance: Instance of main page
|
||||||
|
"""
|
||||||
|
|
||||||
|
from asyncio import ensure_future
|
||||||
|
|
||||||
|
print_substep("Post is NSFW. You are spicy...")
|
||||||
|
# To await indirectly reload
|
||||||
|
navigation = ensure_future(page_instance.waitForNavigation())
|
||||||
|
|
||||||
|
# Triggers indirectly reload
|
||||||
|
await self.click(
|
||||||
|
page_instance,
|
||||||
|
'//button[text()="Yes"]',
|
||||||
|
find_options={"timeout": 5000},
|
||||||
|
)
|
||||||
|
|
||||||
|
# Await reload
|
||||||
|
await navigation
|
||||||
|
|
||||||
|
await self.click(
|
||||||
|
page_instance,
|
||||||
|
'//button[text()="Click to see nsfw"]',
|
||||||
|
find_options={"timeout": 5000},
|
||||||
|
)
|
||||||
|
|
||||||
|
async def __collect_comment(
|
||||||
|
self,
|
||||||
|
comment_obj: dict,
|
||||||
|
filename_idx: int,
|
||||||
|
) -> None:
|
||||||
|
"""
|
||||||
|
Makes a screenshot of the comment
|
||||||
|
|
||||||
|
Args:
|
||||||
|
comment_obj: prew comment object
|
||||||
|
filename_idx: index for the filename
|
||||||
|
"""
|
||||||
|
comment_page = await self.browser.newPage()
|
||||||
|
await comment_page.goto(f'https://reddit.com{comment_obj["comment_url"]}')
|
||||||
|
|
||||||
|
# Translates submission' comment
|
||||||
|
if self.post_lang:
|
||||||
|
comment_tl = ts.google(
|
||||||
|
comment_obj["comment_body"],
|
||||||
|
to_language=self.post_lang,
|
||||||
|
)
|
||||||
|
await comment_page.evaluate(
|
||||||
|
'([comment_id, comment_tl]) => document.querySelector(`#t1_${comment_id} > div:nth-child(2) > div > div[data-testid="comment"] > div`).textContent = comment_tl', # noqa
|
||||||
|
[comment_obj["comment_id"], comment_tl],
|
||||||
|
)
|
||||||
|
|
||||||
|
await self.screenshot(
|
||||||
|
comment_page,
|
||||||
|
f"//div[@id='t1_{comment_obj['comment_id']}']",
|
||||||
|
{"path": f"assets/temp/png/comment_{filename_idx}.png"},
|
||||||
|
)
|
||||||
|
|
||||||
|
# WIP TODO test it
|
||||||
|
async def __collect_story(
|
||||||
|
self,
|
||||||
|
main_page: PageCls,
|
||||||
|
):
|
||||||
|
# Translates submission text
|
||||||
|
if self.post_lang:
|
||||||
|
story_tl = ts.google(
|
||||||
|
self.reddit_object["thread_post"],
|
||||||
|
to_language=self.post_lang,
|
||||||
|
)
|
||||||
|
split_story_tl = story_tl.split('\n')
|
||||||
|
|
||||||
|
await main_page.evaluate(
|
||||||
|
"(split_story_tl) => split_story_tl.map(function(element, i) { return [element, document.querySelectorAll('[data-test-id=\"post-content\"] > [data-click-id=\"text\"] > div > p')[i]]; }).forEach(mappedElement => mappedElement[1].textContent = mappedElement[0])", # noqa
|
||||||
|
split_story_tl,
|
||||||
|
)
|
||||||
|
|
||||||
|
await self.screenshot(
|
||||||
|
main_page,
|
||||||
|
"//div[@data-test-id='post-content']//div[@data-click-id='text']",
|
||||||
|
{"path": "assets/temp/png/story_content.png"},
|
||||||
|
)
|
||||||
|
|
||||||
|
async def download(
|
||||||
|
self,
|
||||||
|
):
|
||||||
|
"""
|
||||||
|
Downloads screenshots of reddit posts as seen on the web. Downloads to assets/temp/png
|
||||||
|
"""
|
||||||
|
print_step("Downloading screenshots of reddit posts...")
|
||||||
|
|
||||||
|
print_substep("Launching Headless Browser...")
|
||||||
|
await self.get_browser()
|
||||||
|
|
||||||
|
# ! Make sure the reddit screenshots folder exists
|
||||||
|
Path("assets/temp/png").mkdir(parents=True, exist_ok=True)
|
||||||
|
|
||||||
|
# Get the thread screenshot
|
||||||
|
reddit_main = await self.browser.newPage()
|
||||||
|
await reddit_main.goto(self.reddit_object["thread_url"]) # noqa
|
||||||
|
|
||||||
|
if settings.config["settings"]["theme"] == "dark":
|
||||||
|
await self.__dark_theme(reddit_main)
|
||||||
|
|
||||||
|
if self.reddit_object["is_nsfw"]:
|
||||||
|
# This means the post is NSFW and requires to click the proceed button.
|
||||||
|
await self.__close_nsfw(reddit_main)
|
||||||
|
|
||||||
|
# Translates submission title
|
||||||
|
if self.post_lang:
|
||||||
|
print_substep("Translating post...")
|
||||||
|
texts_in_tl = ts.google(
|
||||||
|
self.reddit_object["thread_title"],
|
||||||
|
to_language=self.post_lang,
|
||||||
|
)
|
||||||
|
|
||||||
|
await reddit_main.evaluate(
|
||||||
|
f"(texts_in_tl) => document.querySelector('[data-test-id=\"post-content\"] > div:nth-child(3) > div > div').textContent = texts_in_tl", # noqa
|
||||||
|
texts_in_tl,
|
||||||
|
)
|
||||||
|
else:
|
||||||
|
print_substep("Skipping translation...")
|
||||||
|
|
||||||
|
# No sense to move it to common.py
|
||||||
|
async_tasks_primary = ( # noqa
|
||||||
|
[
|
||||||
|
self.__collect_comment(self.reddit_object["comments"][idx], idx) for idx in
|
||||||
|
self.screenshot_idx
|
||||||
|
]
|
||||||
|
if not self.story_mode
|
||||||
|
else [
|
||||||
|
self.__collect_story(reddit_main)
|
||||||
|
]
|
||||||
|
)
|
||||||
|
|
||||||
|
async_tasks_primary.append(
|
||||||
|
self.screenshot(
|
||||||
|
reddit_main,
|
||||||
|
f"//div[@data-testid='post-container']",
|
||||||
|
{"path": "assets/temp/png/title.png"},
|
||||||
|
)
|
||||||
|
)
|
||||||
|
|
||||||
|
for idx, chunked_tasks in enumerate(
|
||||||
|
[chunk for chunk in common.chunks(async_tasks_primary, 10)],
|
||||||
|
start=1,
|
||||||
|
):
|
||||||
|
chunk_list = async_tasks_primary.__len__() // 10 + (1 if async_tasks_primary.__len__() % 10 != 0 else 0)
|
||||||
|
for task in track(
|
||||||
|
as_completed(chunked_tasks),
|
||||||
|
description=f"Downloading comments: Chunk {idx}/{chunk_list}",
|
||||||
|
total=chunked_tasks.__len__(),
|
||||||
|
):
|
||||||
|
await task
|
||||||
|
|
||||||
|
print_substep("Comments downloaded Successfully.", style="bold green")
|
||||||
|
await self.close_browser()
|
@ -0,0 +1,23 @@
|
|||||||
|
from typing import Union
|
||||||
|
|
||||||
|
from webdriver.pyppeteer import RedditScreenshot as Pyppeteer
|
||||||
|
from webdriver.playwright import RedditScreenshot as Playwright
|
||||||
|
|
||||||
|
|
||||||
|
def screenshot_factory(
|
||||||
|
driver: str,
|
||||||
|
) -> Union[type(Pyppeteer), type(Playwright)]:
|
||||||
|
"""
|
||||||
|
Factory for webdriver
|
||||||
|
Args:
|
||||||
|
driver: (str) Name of a driver
|
||||||
|
|
||||||
|
Returns:
|
||||||
|
Webdriver instance
|
||||||
|
"""
|
||||||
|
web_drivers = {
|
||||||
|
"pyppeteer": Pyppeteer,
|
||||||
|
"playwright": Playwright,
|
||||||
|
}
|
||||||
|
|
||||||
|
return web_drivers[driver]
|
Loading…
Reference in new issue