Fix parsing of non-utf8 encoded setup.py files

- Add encoding detection fallback for ast paring of setup.py
- Fixes #205

Signed-off-by: Dan Ryan <dan.ryan@canonical.com>

.pre-commit-config.yaml

@@ -17,8 +17,6 @@ repos:
         args: [
           --application-directories=src/requirementslib,
           --settings-path=./,
-          # --exclude=tests/.*\.py,
-          # --exclude=src/requirementslib/models/setup_info.py
         ]
 
   - repo: https://github.com/timothycrosley/isort

news/205.bugfix.rst

@@ -0,0 +1 @@
+Fixed an issue which prevented successful parsing of ``setup.py`` files which were not ``utf-8`` encoded.

setup.cfg

@@ -127,7 +127,7 @@ not_skip = __init__.py
 line_length = 90
 indent = '    '
 multi_line_output = 3
-known_third_party = appdirs,attr,cached_property,distlib,environ,hypothesis,invoke,orderedmultidict,packaging,parver,pep517,pip_shims,pkg_resources,plette,pyparsing,pytest,requests,setuptools,six,tomlkit,towncrier,urllib3,vistir
+known_third_party = appdirs,attr,cached_property,chardet,distlib,environ,hypothesis,invoke,orderedmultidict,packaging,parver,pep517,pip_shims,pkg_resources,plette,pyparsing,pytest,requests,setuptools,six,tomlkit,towncrier,urllib3,vistir
 known_first_party = requirementslib,tests
 combine_as_imports=True
 include_trailing_comma = True

src/requirementslib/models/setup_info.py

@@ -13,6 +13,7 @@
 from functools import partial
 
 import attr
+import chardet
 import packaging.specifiers
 import packaging.utils
 import packaging.version
@@ -908,7 +909,15 @@ def ast_parse_attribute_from_file(path, attribute):
 
 def ast_parse_file(path):
     # type: (S) -> Analyzer
-    tree = ast.parse(read_source(path))
+    try:
+        tree = ast.parse(read_source(path))
+    except SyntaxError:
+        # The source may be encoded strangely, e.g. azure-storage
+        # which has a setup.py encoded with utf-8-sig
+        with open(path, "rb") as fh:
+            contents = fh.read()
+        encoding = chardet.detect(contents)["encoding"]
+        tree = ast.parse(contents.decode(encoding))
     ast_analyzer = Analyzer()
     ast_analyzer.visit(tree)
     return ast_analyzer