/
project_tasks.py
336 lines (280 loc) · 11.8 KB
/
project_tasks.py
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
232
233
234
235
236
237
238
239
240
241
242
243
244
245
246
247
248
249
250
251
252
253
254
255
256
257
258
259
260
261
262
263
264
265
266
267
268
269
270
271
272
273
274
275
276
277
278
279
280
281
282
283
284
285
286
287
288
289
290
291
292
293
294
295
296
297
298
299
300
301
302
303
304
305
306
307
308
309
310
311
312
313
314
315
316
317
318
319
320
321
322
323
324
325
326
327
328
329
330
331
332
333
334
335
336
# coding=utf-8
"""
Project tasks
Add the following to your *requirements.txt* file:
* docutils!=0.14rc1; python_version == "[python_versions]"
"""
import ast
import os
from pprint import pformat
import shutil
import textwrap
# noinspection PyUnresolvedReferences
from herringlib.prompt import prompt
# noinspection PyUnresolvedReferences
from herringlib.template import Template
# noinspection PyUnresolvedReferences
from herringlib.venv import VirtualenvInfo
try:
# python3
# noinspection PyUnresolvedReferences,PyCompatibility
from configparser import ConfigParser, NoSectionError
except ImportError:
# python2
# noinspection PyUnresolvedReferences,PyCompatibility
from ConfigParser import ConfigParser, NoSectionError
# noinspection PyUnresolvedReferences
from herring.herring_app import task, HerringFile
# noinspection PyUnresolvedReferences
from herringlib.simple_logger import info, debug
# noinspection PyUnresolvedReferences
from herringlib.local_shell import LocalShell
# noinspection PyUnresolvedReferences
from herringlib.requirements import Requirements, Requirement
# noinspection PyUnresolvedReferences
from herringlib.project_settings import Project, ATTRIBUTES
missing_modules = []
def value_from_setup_py(arg_name):
"""
Use AST to find the name value in the setup() call in setup.py.
Only works for key=string arguments to setup().
:param arg_name: the keyword argument name passed to the setup() call in setup.py.
:type arg_name: str
:returns: the name value or None
:rtype: str|None
"""
setup_py = 'setup.py'
if os.path.isfile(setup_py):
# scan setup.py for a call to 'setup'.
tree = ast.parse(''.join(open(setup_py)))
call_nodes = [node.value for node in tree.body if type(node) == ast.Expr and type(node.value) == ast.Call]
# noinspection PyShadowingNames
def is_setup(call_node):
try:
return call_node.func.id == 'setup'
except AttributeError as ex:
try:
return call_node.func.value.id == 'setup'
except AttributeError as ex:
print(str(ex))
print(ast.dump(call_node.func))
print(ast.dump(call_node))
keywords = [call_node.keywords for call_node in call_nodes if is_setup(call_node)]
# now setup() takes keyword arguments so scan them looking for key that matches the given arg_name,
# then return the keyword's value
for keyword in keywords:
for keyword_arg in keyword:
if keyword_arg.arg == arg_name:
if hasattr(keyword_arg.value, 's'):
return keyword_arg.value.s
# didn't find it
return None
def _project_defaults():
"""
Get the project defaults from (in order of preference):
* setup.py,
* kwargs,
* herring config file,
* environment variables,
* default values.
:return: dictionary of defaults
:rtype: dict[str,str]
"""
defaults = {
'package': os.path.basename(os.path.abspath(os.curdir)),
'name': os.path.basename(os.path.abspath(os.curdir)).capitalize(),
'description': 'The greatest project there ever was or will be!',
'author': 'author',
'title': os.path.basename(os.path.abspath(os.curdir)).capitalize(),
}
if 'USER' in os.environ:
defaults['author'] = os.environ['USER']
defaults['author_email'] = '{author}@example.com'.format(author=defaults['author'])
# override defaults from herringfile
# for key in ['name', 'author', 'author_email', 'description']:
# attributes = Project.attributes()
# for key in [key for key in attributes.keys() if attributes[key] is not None]:
# # noinspection PyBroadException
# try:
# value = getattr(Project, key, None)
# if value is not None:
# defaults[key] = value
# except:
# pass
# override defaults from any config files
settings = HerringFile.settings
if settings is not None:
config = ConfigParser()
config.read(settings.config_files)
for section in ['project']:
try:
defaults.update(dict(config.items(section)))
except NoSectionError:
pass
# override defaults from kwargs
for key in task.kwargs:
defaults[key] = task.kwargs[key]
# override defaults from setup.py
for key in ['name', 'author', 'author_email', 'description']:
value = value_from_setup_py(key)
if value is not None:
defaults[key] = value
# now add any attributes that are not already in defaults
for key in ATTRIBUTES:
if key not in defaults:
value = getattr(Project, key, None)
if value is not None:
defaults[key] = value
else:
print("{key}:None".format(key=key))
return defaults
@task(namespace='project', help='Available options: --name, --package, --author, --author_email, --description',
kwargs=['name', 'package', 'author', 'author_email', 'description'], configured='no')
def init():
"""
Initialize a new python project with default files. Default values from herring.conf and directory name.
"""
defaults = _project_defaults()
if Project.prompt:
defaults['name'] = prompt("Enter the project's name:", defaults['name'])
defaults['package'] = prompt("Enter the project's package:", defaults['package'])
defaults['author'] = prompt("Enter the project's author:", defaults['author'])
defaults['author_email'] = prompt("Enter the project's author's email:", defaults['author_email'])
defaults['description'] = prompt("Enter the project's description:", defaults['description'])
# print("defaults:\n{defaults}".format(defaults=pformat(defaults)))
if Project.use_templates:
template = Template()
for template_dir in [os.path.abspath(os.path.join(herringlib, 'herringlib', 'templates'))
for herringlib in HerringFile.herringlib_paths]:
info("template directory: %s" % template_dir)
# noinspection PyArgumentEqualDefault
template.generate(template_dir, defaults, overwrite=False)
@task(namespace='project')
def update():
"""
Regenerate files (except herringfile) from current templates.
Delete the file(s) you want to update, then run this task.
"""
if Project.use_templates:
defaults = _project_defaults()
template = Template()
for template_dir in [os.path.abspath(os.path.join(herringlib, 'herringlib', 'templates'))
for herringlib in HerringFile.herringlib_paths]:
info("template directory: %s" % template_dir)
# noinspection PyArgumentEqualDefault
template.generate(template_dir, defaults, overwrite=False)
@task(namespace='project', configured='optional')
def show():
"""Show all project settings"""
info(str(Project))
@task(namespace='project', configured='optional')
def describe():
"""Show all project settings with descriptions"""
keys = Project.__dict__.keys()
for key in sorted(keys):
value = Project.__dict__[key]
if key in ATTRIBUTES:
attrs = ATTRIBUTES[key]
required = False
if 'required' in attrs:
if attrs['required']:
required = True
if 'help' in attrs:
info("# {key}".format(key=key))
if required:
info("# REQUIRED")
for line in textwrap.wrap(attrs['help'], width=100):
info("# {line}".format(line=line))
info("# '{key}': '{value}'".format(key=key, value=value))
info('')
else:
info("'{key}': '{value}'".format(key=key, value=value))
def _pip_list():
names = []
# noinspection PyBroadException
try:
# idiotic python setup tools creates empty egg directory in project that then causes pip to blow up.
# Wonderful python tools in action!
# so lets remove the stupid egg directory so we can use pip to get a listing of installed packages.
egg_info_dir = "{name}.egg-info".format(name=Project.name)
if os.path.exists(egg_info_dir):
shutil.rmtree(egg_info_dir)
with LocalShell() as local:
# if 'VIRTUAL_ENV' in os.environ:
# pip = os.path.join(os.environ['VIRTUAL_ENV'], 'bin', 'pip')
# info("PATH={path}".format(path=os.environ['PATH']))
# info(pip)
pip = local.system('which pip || which pip3', verbose=False).strip()
# info(pip)
# info("pip version: {ver}".format(ver=local.system('{pip} --version'.format(pip=pip))))
pip_list_output = local.run('{pip} list'.format(pip=pip))
# info(pip_list_output)
lines = pip_list_output.split("\n")
names = [line.split(" ")[0].lower().encode('ascii', 'ignore') for line in lines if line.strip()]
except Exception:
pass
return names
# noinspection PyArgumentEqualDefault
__pip_list = [pkg.decode('utf-8') for pkg in _pip_list()]
def packages_required(package_names):
"""
Check that the given packages are installed.
:param package_names: the package names
:type package_names: list
:return: asserted if all the packages are installed
:rtype: bool
"""
# info("packages_required(%s)" % repr(package_names))
# noinspection PyBroadException
try:
result = True
# info(package_names)
# info(__pip_list)
for requirement in [Requirement(name) for name in package_names]:
if requirement.supported_python():
pkg_name = requirement.package
if pkg_name.lower() not in __pip_list:
try:
# info('__import__("{name}")'.format(name=pkg_name))
__import__(pkg_name)
except ImportError:
info(pkg_name + " not installed!")
missing_modules.append(pkg_name)
result = False
return result
except Exception:
return False
@task(configured='optional')
def show_missing():
"""Show modules that if installed would enable additional tasks."""
if missing_modules:
info("The following modules are currently not installed and would enable additional tasks:")
for pkg_name in missing_modules:
info(' ' + pkg_name)
# noinspection PyArgumentEqualDefault
@task(namespace='project', private=False)
def check_requirements():
""" Checks that herringfile and herringlib/* required packages are in requirements.txt file """
debug("check_requirements")
needed = Requirements(Project).find_missing_requirements()
if needed:
info("Please add the following to your %s file:\n" % 'requirements.txt')
info("\n".join(str(needed)))
else:
info("Your %s includes all known herringlib task requirements" % 'requirements.txt')
@task(namespace='project', configured='required')
def environment():
""" Display project environment """
venvs = VirtualenvInfo('python_versions')
site_packages_cmdline = "python -c 'from distutils.sysconfig import get_python_lib; print(get_python_lib())'"
project_env = {}
if not venvs.in_virtualenv and venvs.defined:
for venv_info in venvs.infos():
site_packages = venv_info.run(site_packages_cmdline).strip().splitlines()[2]
project_env[venv_info.venv + ': site-packages'] = site_packages
else:
with LocalShell() as local:
site_packages = local.system(site_packages_cmdline).strip()
project_env['site-packages'] = site_packages
info(pformat(project_env))
return project_env