/
create_scripts.py
164 lines (133 loc) · 6.26 KB
/
create_scripts.py
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
"""
Copyright (c) 2010-2022, Delft University of Technology. All rights reserved. This file is part of the Tudat. Redistribution and use in source and
binary forms, with or without modification, are permitted exclusively
under the terms of the Modified BSD license. You should have received
a copy of the license with this file. If not, please or visit:
http://tudat.tudelft.nl/LICENSE.
"""
# PLEASE NOTE:
# This script is NOT a tudatpy example.
# It is a script to clean the .py files that are generated from Jupyter notebooks, using the following option: File > Download as > Python (.py)
# Running it will automatically edit all the .py example files (please check the changes made before pushing them to the repository).
# Standard library imports
import re
import glob
import subprocess
# Other imports
from tqdm import tqdm
# Utilities
def request_confirmation(message):
message = f'{message} [y/N]'
width = max(60, len(message)+20)
return input(
f'{"="*width}\n{message:^{width}}\n{"="*width}\n'
).strip().lower() == 'y'
"""
Use the find command line utility to find the paths of all
notebooks in this repository and store them in a list
"""
example_notebooks = glob.glob('**/*.ipynb', recursive=True)
example_scripts = [notebook.replace('.ipynb', '.py') for notebook in example_notebooks]
all_python_files = glob.glob('**/*.py', recursive=True)
"""
Transform each notebook into a python script using
the jupyter nbconvert command line utility
"""
def generate_script(notebook):
subprocess.run(['jupyter', 'nbconvert', '--to', 'script', notebook])
if request_confirmation('Regenerate Python scripts from Jupyter notebooks?'):
# Generate the python scripts
for notebook in tqdm(example_notebooks): generate_script(notebook)
# Assert that all the notebooks were converted to python scripts
assert all([script in all_python_files for script in example_scripts]), \
f'Unsuccessful: not all notebooks were converted to python scripts. Failed conversions:\n' + \
'\n'.join([script for script in example_scripts if script not in all_python_files])
else:
# If there are missing scripts
if not all([script in all_python_files for script in example_scripts]):
# Generate the missing python scripts
for script in [script for script in example_scripts if script not in all_python_files]:
generate_script(script)
"""
Clean up the python scripts
"""
for example_python_script in example_scripts:
print(f'Cleaning example: {example_python_script}')
with open(example_python_script, "r+") as file:
# Read example
example_content = file.readlines()
# Remove file type and encoding
if "!/usr/bin/env python" in example_content[0]:
example_content = example_content[3:]
# State
checking_comment_end = False
skip_next = False
# Indentation
indentation = ''
# Go trough each line in the example
for i, line in enumerate(example_content):
if skip_next:
skip_next = False
continue
# --> Remove the "In[x]" notebook inputs
if "In[" in line:
# Also remove the two lines after
[example_content.pop(i) for _ in range(3)]
# --> End of MD cell
elif checking_comment_end:
# --> End of cell: if the line is empty, the markdown cell is finished
if line == "\n":
# Add """ to close the string comment, then an empty line
example_content[i] = "\"\"\"\n"
example_content.insert(i+1, "\n")
checking_comment_end = False
# --> Second title: detect if we have a second title in the same markdown cell, and mark the separation
elif "##" in line:
example_content[i] = line.replace("# ", "", 1)
example_content.insert(i, "\"\"\"\n\n")
example_content.insert(i+2, "\"\"\"\n")
skip_next = True
# If we are still in the markdown cell, remove the simple # that indicates a comment line
else:
example_content[i] = line.replace("# ", "", 1)
# --> Start of MD cell: if the line starts with # #, we are in a markdown cell that starts with a title
elif "# #" in line:
# Replace the first line to keep the title with a comment #
example_content[i] = line.replace("# ", "", 1)
example_content.insert(i+1, "\"\"\"\n")
if example_content[i+2] == "\n":
example_content.pop(i+2)
checking_comment_end = True # Start looking for the end of the cell
# --> Remove the lines that made the plots interactive
elif "# make plots interactive" in line:
[example_content.pop(i) for _ in range(2)]
# We're in a code cell, so we record the indentation level
else:
# Retrieve the last non-empty line
last_nonempty_line = next(line for line in example_content[i-1::-1] if re.match(r'^( {4}){0,2}\S', line))
# Keep track of current indentation
indentation = ' ' * (len(last_nonempty_line) - len(last_nonempty_line.lstrip()))
file.seek(0)
file.writelines(example_content)
file.truncate()
"""
Check Python scripts for syntax errors
"""
print('\nChecking Python scripts for syntax errors\n')
for example_python_script in example_scripts:
# Test the example
result = subprocess.run(['python', '-m', 'py_compile', example_python_script],
stdout=subprocess.DEVNULL,
stderr=subprocess.STDOUT)
if result.returncode != 0:
print(f'Unsuccessful: syntax error in example: {example_python_script}')
example_scripts.remove(example_python_script)
print('')
"""
Test python scripts
"""
if request_confirmation('Test generated Python scripts?'):
for example_python_script in example_scripts:
print(f'Testing example: {example_python_script}')
# Test the example
subprocess.run(['python', example_python_script])