From 9c81447096a8e4588f3d97ac146739dabfe5dbe9 Mon Sep 17 00:00:00 2001 From: Drugsosos <44712637+Drugsosos@users.noreply.github.com> Date: Wed, 20 Jul 2022 23:29:00 +0300 Subject: [PATCH] changed locators a bit & updated chunks generator --- webdriver/common.py | 20 ++++++++++++++++---- webdriver/playwright.py | 24 +++++++++++++----------- webdriver/pyppeteer.py | 28 +++++++++++++++------------- 3 files changed, 44 insertions(+), 28 deletions(-) diff --git a/webdriver/common.py b/webdriver/common.py index 5e544f8..0061b3b 100644 --- a/webdriver/common.py +++ b/webdriver/common.py @@ -66,7 +66,19 @@ def catch_exception( # Lots of tabs - lots of memory # chunk needed to minimize memory required -def chunks(lst, n): - """Yield successive n-sized chunks from list.""" - for i in range(0, len(lst), n): - yield lst[i:i + n] +def chunks( + array: list, + size: int, +): + """ + Yield successive n-sized chunks from list. + + Args: + array: List to be chunked + size: size of a chunk + + Returns: + Generator with chunked list + """ + for i in range(0, len(array), size): + yield array[i:i + size] diff --git a/webdriver/playwright.py b/webdriver/playwright.py index f7145bb..b9bd0fb 100644 --- a/webdriver/playwright.py +++ b/webdriver/playwright.py @@ -14,7 +14,6 @@ from utils.console import print_step, print_substep import webdriver.common as common - common.default_exception = TimeoutError @@ -188,7 +187,7 @@ class RedditScreenshot(Flaky, Browser): # Triggers indirectly reload await self.click( page_instance, - 'button:has-text("Yes")', + "button:has-text('Yes')", {"timeout": 5000}, ) @@ -197,7 +196,7 @@ class RedditScreenshot(Flaky, Browser): await self.click( page_instance, - 'button:has-text("Click to see nsfw")', + "button:has-text('Click to see nsfw')", {"timeout": 5000}, ) @@ -229,7 +228,7 @@ class RedditScreenshot(Flaky, Browser): await self.screenshot( comment_page, - f"[data-testid='post-container']", + f"id=t1_{comment_obj['comment_id']}", {"path": f"assets/temp/png/comment_{filename_idx}.png"}, ) @@ -246,20 +245,23 @@ class RedditScreenshot(Flaky, Browser): ) split_story_tl = story_tl.split('\n') await main_page.evaluate( - # Find all elements - 'var elements = document.querySelectorAll(`[data-test-id="post-content"]' - ' > [data-click-id="text"] > div > p`);' + # Find all elements with story text + "const elements = document.querySelectorAll('[data-test-id=\"post-content\"]" + " > [data-click-id=\"text\"] > div > p');" # Set array with translated text - f"var texts = {split_story_tl};" + f"const texts = {split_story_tl};" # Map 2 arrays together - "var text_map = texts.map(function(e, i) { return [e, elements[i]]; });" + "const concat = (element, i) => [element, elements[i]];" + "const mappedTexts = texts.map(concat);" # Change text on the page - "for (i = 0; i < text_map.length; ++i) { text_map[i][1].textContent = text_map[i][0] ; };" + "for (i = 0; i < mappedTexts.length; ++i) {" + "mappedTexts[i][1].textContent = mappedTexts[i][0];" + "};" ) await self.screenshot( main_page, - '[data-test-id="post-content"] > [data-click-id="text"]', + "data-test-id='post-content' > data-click-id='text'", {"path": "assets/temp/png/story_content.png"}, ) diff --git a/webdriver/pyppeteer.py b/webdriver/pyppeteer.py index b85a53d..b2c1e2a 100644 --- a/webdriver/pyppeteer.py +++ b/webdriver/pyppeteer.py @@ -18,7 +18,6 @@ from typing import Optional import webdriver.common as common - common.default_exception = BrowserTimeoutError @@ -183,27 +182,27 @@ class RedditScreenshot(Browser, Wait): await self.click( page_instance, - "//*[contains(@class, 'header-user-dropdown')]", + "//div[@class='header-user-dropdown']", find_options={"timeout": 5000}, ) # It's normal not to find it, sometimes there is none :shrug: await self.click( page_instance, - "//*[contains(text(), 'Settings')]/ancestor::button[1]", + "//span[text()='Settings']/ancestor::button[1]", find_options={"timeout": 5000}, ) await self.click( page_instance, - "//*[contains(text(), 'Dark Mode')]/ancestor::button[1]", + "//span[text()='Dark Mode']/ancestor::button[1]", find_options={"timeout": 5000}, ) # Closes settings await self.click( page_instance, - "//*[contains(@class, 'header-user-dropdown')]", + "//div[@class='header-user-dropdown']", find_options={"timeout": 5000}, ) @@ -268,7 +267,7 @@ class RedditScreenshot(Browser, Wait): await self.screenshot( comment_page, - f"//*[contains(@id, 't1_{comment_obj['comment_id']}')]", + f"//div[@id='t1_{comment_obj['comment_id']}']", {"path": f"assets/temp/png/comment_{filename_idx}.png"}, ) @@ -285,15 +284,18 @@ class RedditScreenshot(Browser, Wait): ) split_story_tl = story_tl.split('\n') await main_page.evaluate( - # Find all elements - 'var elements = document.querySelectorAll(`[data-test-id="post-content"]' - ' > [data-click-id="text"] > div > p`);' + # Find all elements with story text + "const elements = document.querySelectorAll('[data-test-id=\"post-content\"]" + " > [data-click-id=\"text\"] > div > p');" # Set array with translated text - f"var texts = {split_story_tl};" + f"const texts = {split_story_tl};" # Map 2 arrays together - "var text_map = texts.map(function(e, i) { return [e, elements[i]]; });" + "const concat = (element, i) => [element, elements[i]];" + "const mappedTexts = texts.map(concat);" # Change text on the page - "for (i = 0; i < text_map.length; ++i) { text_map[i][1].textContent = text_map[i][0] ; };" + "for (i = 0; i < mappedTexts.length; ++i) {" + "mappedTexts[i][1].textContent = mappedTexts[i][0];" + "};" ) await self.screenshot( @@ -359,7 +361,7 @@ class RedditScreenshot(Browser, Wait): async_tasks_primary.append( self.screenshot( reddit_main, - f'//*[@data-testid="post-container"]', + f"//div[@data-testid='post-container']", {"path": "assets/temp/png/title.png"}, ) )