pyproject.toml

[project]

name = "grab"
version = "0.6.41"
description = "Web scraping framework"
readme = "README.md"
requires-python = ">=3.8"
license = {"file" = "LICENSE"}
keywords = ["web crawling", "web scraping", "web crawler", "web scraper", "lxml", "xpath", "html parsing"]
authors = [
    {name = "Gregory Petukhov", email = "lorien@lorien.name"}
]
# https://pypi.org/pypi?%3Aaction=list_classifiers
classifiers = [
    "Programming Language :: Python",
    "Programming Language :: Python :: 3",
    "Programming Language :: Python :: 3.8",
    "Programming Language :: Python :: 3.9",
    "Programming Language :: Python :: 3.10",
    "Programming Language :: Python :: 3.11",
    "License :: OSI Approved :: MIT License",
    "Development Status :: 4 - Beta",
    "Environment :: Console",
    "Intended Audience :: Developers",
    "Operating System :: OS Independent",
    "Topic :: Software Development :: Libraries :: Application Frameworks",
    "Topic :: Software Development :: Libraries :: Python Modules",
    "Topic :: Internet :: WWW/HTTP",
    "Typing :: Typed",
]
dependencies = [
    "proxylist>=0.2.1",
    "unicodec>=0.0.8",
    "procstat>=0.0.7",
    "user_agent",
    "selection>=0.0.21",
    "lxml",
    "urllib3[socks]",
    "certifi",
]

[project.urls]
homepage = "http://github.com/lorien/grab"

[project.optional-dependencies]
pyquery = ["pyquery"]
cssselect = ["cssselect"]

[build-system]
requires = ["setuptools"]
build-backend = "setuptools.build_meta"

[tool.setuptools]
packages=["grab", "grab.spider", "grab.spider.queue_backend", "grab.spider.service", "grab.util"]

[tool.setuptools.package-data]
"*" = ["py.typed"]

[tool.isort]
profile = "black"
line_length = 88
known_third_party = ["unicodec"]
# skip_gitignore = true # throws errors in stderr when ".git" dir does not exist

[tool.bandit]
# B101 assert_used
# B410 Using HtmlElement to parse untrusted XML data
# B320 Using lxml.etree.parse to parse untrusted XML data
# B310 Audit url open for permitted schemes
# B301 Pickle and modules that wrap it can be unsafe
# B403 Consider possible security implications associated with pickle module 
# B324  Use of weak MD4, MD5, or SHA1 hash for security
# B303 Use of insecure MD2, MD4, MD5, or SHA1 hash function
skips = ["B101", "B410", "B320", "B310", "B301", "B403", "B324", "B303"]

[[tool.mypy.overrides]]
module = "pyquery"
ignore_missing_imports = true

[[tool.mypy.overrides]]
module = ["user_agent", "fastrq.priorityqueue"]
ignore_missing_imports = true

[tool.pylint.main]
jobs=4
extension-pkg-whitelist="lxml"
disable="missing-docstring,broad-except,too-few-public-methods,consider-using-f-string,fixme"
variable-rgx="[a-z_][a-z0-9_]{1,30}$"
attr-rgx="[a-z_][a-z0-9_]{1,30}$"
argument-rgx="[a-z_][a-z0-9_]{1,30}$"
max-line-length=88
max-args=9
load-plugins=[
    "pylint.extensions.check_elif",
    "pylint.extensions.comparetozero",
    "pylint.extensions.comparison_placement",
    "pylint.extensions.consider_ternary_expression",
    "pylint.extensions.docstyle",
    "pylint.extensions.emptystring",
    "pylint.extensions.for_any_all",
    "pylint.extensions.overlapping_exceptions",
    "pylint.extensions.redefined_loop_name",
    "pylint.extensions.redefined_variable_type",
    "pylint.extensions.set_membership",
    "pylint.extensions.typing",
]

[tool.pytest.ini_options]
testpaths = ["tests"]
addopts = "--tb=short"

[tool.ruff]
pylint.max-args=9
select = ["ALL"]
ignore = [
    "A003", # Class attribute `type` is shadowing a python builtin
    "ANN101", # Missing type annotation for `self` in method
    "ANN401", # Dynamically typed expressions (typing.Any) are disallowed
    "BLE001", # Do not catch blind exception: `Exception`
    "COM812", # Trailing comma missing
    "D100", # Missing docstring in public module
    "D101", # Missing docstring in public class
    "D102", # Missing docstring in public method1
    "D103", # Missing docstring in public function
    "D104", # Missing docstring in public package
    "D105", # Missing docstring in magic method,
    "D107", # Missing docstring in `__init__`
    "D203", # 1 blank line required before class docstring
    "D213", # Multi-line docstring summary should start at the second line
    "EM101", # Check for raw usage of a string literal in Exception raising
    "EM102", # Check for raw usage of an f-string literal in Exception raising
    "EM103", # Check for raw usage of .format on a string literal in Exception raising
    "F401", # Imported but unused
    "FBT", # Boolean arg/value in function definition
    "PTH", # A plugin finding use of functions that can be replaced by pathlib module
    "S101", # Use of `assert` detected
    "T201", # print found	
    "T203", # pprint found
    "TCH", # Move import into a type-checking block
    "TRY003", # Avoid specifying long messages outside the exception class
    "UP032", # Use f-string instead of format call
    "ERA001", # Found commented-out code
    "RUF001", # String contains ambiguous unicode character
    "ANN102", # Missing type annotation for `cls` in classmethod
    # TODO: remove following rules
    "PT009", # Use a regular `assert` instead of unittest-style `assertEqual`
    "RUF012", # Mutable class attributes should be annotated with `typing.ClassVar`
    "TD002", # Missing author in TODO
    "TD003", # Missing issue link on the line following this TODO
    "FIX002", # Line contains TODO
    "S320", # Using `xml` to parse untrusted data is known to be vulnerable to XML attacks
    "D416", # Section name should end with a colon
    "D417", # Missing argument descriptions in the docstring
    "D407", # Missing dashed underline after section
    "D406", # Section name should end with a newline
    "UP012", # Unnecessary call to `encode` as UTF-8
    "S310", # Audit URL open for permitted schemes
    "D106", # Missing docstring in public nested class
    "PGH003", # Use specific rule codes when ignoring type issues
]
target-version = "py38"
extend-exclude = ["var"]