Commit
This commit does not belong to any branch on this repository, and may belong to a fork outside of the repository.
Merge pull request #49 from hhslepicka/fixes-and-features
New Utilities, Improved Docs and Bug Fix
- Loading branch information
Showing
8 changed files
with
349 additions
and
21 deletions.
There are no files selected for viewing
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
Original file line number | Diff line number | Diff line change |
---|---|---|
@@ -0,0 +1,90 @@ | ||
import collections | ||
import string | ||
from typing import Dict, List | ||
from selenium.webdriver.remote.webelement import WebElement | ||
|
||
|
||
def data_from_row(row: WebElement, cell_tag="td") -> List[str]: | ||
"""Extract data from a row and return it as a list. | ||
Args: | ||
row (WebElement): The row element. | ||
cell_tag (str, optional): The HTML tag associated with the row cells. Defaults to "td". | ||
Returns: | ||
list: List of strings with the contents. | ||
""" | ||
return [ | ||
col.text for col in row.find_elements_by_tag_name(cell_tag) | ||
] | ||
|
||
|
||
def sanitize_header(labels: List[str]): | ||
"""Sanitize header labels.""" | ||
# Handle Treat Empty Header | ||
for idx, label in enumerate(labels): | ||
if label.strip(): | ||
# make it lowercase | ||
label = label.lower() | ||
|
||
# remove punctuations | ||
label = ''.join([l for l in label if l not in string.punctuation]) # noqa: E741 | ||
|
||
# replace spaces with underscores | ||
label = label.replace(" ", "_") | ||
else: | ||
label = f"col_{idx}" | ||
labels[idx] = label | ||
|
||
# Deduplicate by adding _1, _2, _3 to repeated labels | ||
counts = {k: v for k, v in collections.Counter(labels).items() if v > 1} | ||
for i in reversed(range(len(labels))): | ||
item = labels[i] | ||
if item in counts and counts[item]: | ||
labels[i] = f"{item}_{counts[item]}" | ||
counts[item] -= 1 | ||
|
||
return labels | ||
|
||
|
||
def table_to_dict(table: WebElement, has_header: bool = True, | ||
skip_rows: int = 0, header_tag: str = "th") -> List[Dict]: | ||
"""Convert a table WebElement to a dict of lists. | ||
Args: | ||
table (WebElement): The table element. | ||
has_header (bool, optional): Whether or not to parse a header. Defaults to True. | ||
skip_rows (int, optional): Number of rows to skip from the top. Defaults to 0. | ||
header_tag (str, optional): The HTML tag associated with the header cell. Defaults to "th". | ||
Returns: | ||
list: List with dict for each row. | ||
""" | ||
|
||
# Collect all rows from table | ||
rows = table.find_elements_by_tag_name("tr") | ||
|
||
# Skip rows if informed | ||
if skip_rows: | ||
rows = rows[skip_rows:] | ||
|
||
# Parse header labels | ||
if has_header: | ||
# Read header labels | ||
labels = data_from_row(rows[0], cell_tag=header_tag) | ||
# Sanitize headers | ||
labels = sanitize_header(labels) | ||
# Skip the header | ||
rows = rows[1:] | ||
else: | ||
# Make up header labels | ||
num_cols = len(rows[0].find_elements_by_tag_name("td")) | ||
labels = [f"col_{i}" for i in range(num_cols)] | ||
|
||
# Assemble output dictionary | ||
out_list = [] | ||
for row in rows: | ||
row_data = data_from_row(row) | ||
out_list.append(dict(zip(labels, row_data))) | ||
|
||
return out_list |
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
Original file line number | Diff line number | Diff line change |
---|---|---|
@@ -1,9 +1,31 @@ | ||
import shutil | ||
import tempfile | ||
|
||
from selenium.webdriver.remote.webelement import WebElement | ||
from selenium.webdriver.support.select import Select | ||
|
||
def cleanup_temp_dir(temp_dir): | ||
|
||
def cleanup_temp_dir(temp_dir: tempfile.TemporaryDirectory) -> None: | ||
""" | ||
Deletes the temporary directory and all its contents. | ||
Args: | ||
temp_dir (tempfile.TemporaryDirectory): The temporary directory to delete. | ||
""" | ||
if temp_dir: | ||
try: | ||
temp_dir.cleanup() | ||
except OSError: | ||
shutil.rmtree(temp_dir.name, ignore_errors=True) | ||
|
||
|
||
def element_as_select(element: WebElement) -> Select: | ||
"""Wraps a WebElement in a Select object. | ||
Args: | ||
element (WebElement): The element to wrap. | ||
Returns: | ||
Select: The Select object. | ||
""" | ||
return Select(element) |
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
Original file line number | Diff line number | Diff line change |
---|---|---|
@@ -0,0 +1,44 @@ | ||
# Interacting with Forms | ||
|
||
When dealing with forms, we often need to fill in the form and submit it. | ||
|
||
While most of the operations are trivial, there are some things that are not such as selecting a select element or dealing with file uploads. | ||
|
||
For that we developed some utilitary functions that you can use. | ||
|
||
## Select Element | ||
|
||
After grabing the element via the `find_element` or `find_elements` functions, we can use the `element_as_select` to convert it into a `Select` object. | ||
|
||
::: botcity.web.util.element_as_select | ||
|
||
### Example usage | ||
|
||
```python | ||
# Import the function | ||
from botcity.web.util import element_as_select | ||
... | ||
# Fetch the select element | ||
element = self.find_element("select", By.TAG_NAME) | ||
# Convert the element into a Select object | ||
select_element = element_as_select(element) | ||
# Select the option based on visible text | ||
select_element.select_by_visible_text("Option 1") | ||
... | ||
``` | ||
|
||
## File Upload | ||
|
||
After grabing the element via the `find_element` or `find_elements` functions, we can use the `set_file_input_element` to assign the file path to the element. | ||
|
||
### Example usage | ||
|
||
```python | ||
from botcity.web import By | ||
... | ||
# Find the input element of type `file` using CSS_SELECTOR. | ||
elem = self.find_element("body > form > input[type=file]", By.CSS_SELECTOR) | ||
# Configure the file to be used when processing the upload | ||
self.set_file_input_element(elem, "./test.txt") | ||
... | ||
``` |
Oops, something went wrong.