/
smartcp.py
executable file
·304 lines (282 loc) · 10 KB
/
smartcp.py
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
232
233
234
235
236
237
238
239
240
241
242
243
244
245
246
247
248
249
250
251
252
253
254
255
256
257
258
259
260
261
262
263
264
265
266
267
268
269
270
271
272
273
274
275
276
277
278
279
280
281
282
283
284
285
286
287
288
289
290
291
292
293
294
295
296
297
298
299
300
301
302
303
304
#!/usr/bin/env python3
# -*- coding: utf-8 -*-
from __future__ import print_function # for python 2 and stderr
from __version__ import version
import yaml
import itertools
import os
import filecmp
import sys
import getopt
import shutil
from subprocess import call
def usage():
print('''\
Usage: {0} [OPTION]... [FILE]...
Read FILE(s) and do smart copies accordingly.
-q, --quiet do not print the stdout of the command executed with -x
with -qq, it does not print stderr neither
-n, --no-copy do not do the copy but execute the command given by -x
-s, --set with the syntax arg=value,
set the argument with lablel arg to value instead of
iterating over all different possible values
-v increment verbose level, -vv gives the most verbose output
-x command execute command in the parent directory of the input
before comparting the input and the output
-g, --google-drive drive push each copied file
-h, --help display this help and exit
--version output version information and exit
With no FILE, or when FILE is -, read standard input.
Examples:
{0} config.yml - */config.yml Do smart copies for config.yml,
then standard input,
then all config.yml in a subdirectory.
{0} Do smart copies for standard output.'''.
format(program_name))
def show_version():
print('''\
{} {}
Copyright (C) 2013 Benoît Legat.
License GPLv3+: GNU GPL version 3 or later <http://gnu.org/licenses/gpl.html>.
This is free software: you are free to change and redistribute it.
There is NO WARRANTY, to the extent permitted by law.
Written by Benoît Legat.'''.format(program_name, version))
def ask_yes_or_no(prompt):
while True:
s = input(prompt)
if len(s) == 0 or s in ["y", "Y","yes","Yes"]:
return True
elif s in ["n","N","no","No"]:
return False
else:
print("Please answer 'y' or 'n'.")
def push_path(google_drive, path):
if google_drive:
# I need to use '{}' with quote in the case there is spaces in "output_path"
call("drive push -no-prompt=true \"{}\"".format(path), shell = True)
def parent_dir_exists(path):
# we take the absolute path to be to transform './...' in '/...'
folder = os.path.dirname(os.path.abspath(path))
last = None
current = folder
while not os.path.exists(current):
last = current
current = os.path.dirname(current)
if current != folder:
if ask_yes_or_no('Create {} in {} ? [Y/n]: '.format(os.path.basename(last), current)):
# /!\ race condition, maybe now it exists :(
os.makedirs(last)
push_path(google_drive, path)
return parent_dir_exists(path)
print_err('There is no {} in {}'.format(os.path.basename(last), current))
return False
return True
def up_to_date(input_path, output_path):
return os.path.exists(output_path) and filecmp.cmp(input_path, output_path)
def get(hash_map, key, raise_err = True):
if key in hash_map:
return hash_map[key]
else:
if raise_err:
print_err('Missing key `{}\' in `{}\''.format(key, hash_map))
sys.exit(1)
else:
return None
def build_path(path_desc, arguments):
if 'path_format' in path_desc:
if 'parameters' in path_desc:
params = [build_path(param, arguments)
for param in path_desc['parameters']]
# Use python3 to avoid problem with accents here
return path_desc['path_format'].format(*params)
else:
return path_desc['path_format']
elif 'mapping' in path_desc:
mapping = path_desc['mapping']
mapstr = dict()
for k in mapping.keys():
mapstr[str(k)] = mapping[k]
mapping = mapstr
key = build_path(get(path_desc, 'key'), arguments)
if key in mapping:
return mapping[key]
else:
return key
elif 'arg' in path_desc:
if arguments:
label = path_desc['arg']
# Si le label est 1 et que les keys sont 'a', 'b',
# la valeur de arguments[1] sera indéterminée,
# ça dépendra de l'ordre des keys du hash. Évitons cela
#if type(label) == int
# Let's check it anyway
if label in arguments:
return arguments[label]
else:
print_err("unknown label `{}', it should be in {}".
format(label, arguments.keys()))
sys.exit(1)
else:
print_err("didn't expect `arg' since \
there is no argument for this client")
sys.exit(1)
else:
print_err("{} should have `arg', `mapping' or `path_format'".
format(path_desc))
sys.exit(1)
def smart_copy(config_file, arg_set, command, quiet, do_copy):
global indent_level
if config_file:
stream = open(config_file, 'r')
print_verbose('Using {}'.format(config_file))
else:
stream = sys.stdin
print_verbose('Using stdin')
indent_level += 1
config = yaml.load(stream)
stream.close()
if not config:
print_err('Empty config file')
sys.exit(1)
input_base = os.path.abspath(get(config, 'input_base'))
for client in get(config, 'clients'):
print_verbose('Updating {}'.format(get(client, 'name')))
indent_level += 1
# If no arguments, simply means path_format do not contain parameters
arguments = get(client, 'arguments', False)
# With &label and *label in YAML,
# the arrays/hash are shared between all clients in python !!!!
# It should not cause any problem here
# (see if I modify config anywhere else)
if arguments:
if type(arguments) != dict:
print_err("arguments which is `{}' should be a hash".format(arguments))
sys.exit(1)
for key, value in arguments.items():
if key in arg_set:
# Solve the problem with non-string
# not being compared properly with components of arg_set which are
# strings
value = [str(arg) for arg in value]
setting = arg_set[key]
if setting in value:
arguments[key] = [setting]
else:
# itertools.product will return an empty iterator
arguments[key] = []
print_verbose("`{}' is not in the list of values of `{}'".format(setting, key))
# So no need to go further
break
# if there is no arguments we just need one loop without any arguments
# so [None] do the trick
# We need the absolute path because if command != None, we will cd
for args_items in (itertools.product(*arguments.values()) if arguments else [None]):
if args_items:
args = dict(zip(arguments.keys(), args_items))
else:
args = None
input_path = os.path.join(input_base,
build_path(get(client, 'input'), args))
if os.path.exists(input_path):
if command:
os.chdir(os.path.dirname(input_path))
# No need to cd back because input_path is absolute
# since input_base is absolute
if quiet >= 1:
dev_null = open(os.devnull, 'wb')
exit_value = call(command, shell = True,
stdout = dev_null if quiet >= 1 else None,
stderr = dev_null if quiet >= 2 else None)
if exit_value != 0:
print_err("`{}' exited with {}. aborting".format(command, exit_value))
sys.exit(1)
output_path = os.path.join(get(config, 'output_base'),
build_path(get(client, 'output'), args))
if parent_dir_exists(output_path):
if up_to_date(input_path, output_path):
print_verbose(u'`{}\' == `{}\''.format(input_path, output_path), 2)
# u is only for python 2
else:
if do_copy:
print_verbose(u'`{}\' -> `{}\''.format(input_path, output_path))
# u is only for python 2
shutil.copyfile(input_path, output_path)
push_path(google_drive, output_path)
else:
print_verbose(u'`{}\' != `{}\''.format(input_path, output_path))
# u is only for python 2
else:
print_verbose(u'`{}\' /\ `{}\''.format(input_path, output_path))
# u is only for python 2
#sys.exit(1)
else:
print_verbose("`{}' not found".format(input_path), 3)
indent_level -= 1
indent_level -= 1
program_name = 'smartcp'
def print_err(message):
print("{0}: {1}".format(program_name, message),
file = sys.stderr)
verbose = 0
indent_level = 0
google_drive = False
def print_verbose(message, level = 1):
if level <= verbose:
print("{}{}".format(" " * indent_level, message))
def main():
arg_set = {}
do_copy = True
command = None
quiet = 0
global verbose
global google_drive
try:
# gnu_getopt allow opts to be after args. For
# $ smartcp.py config.yml -v
# gnu_getopt will consider -v as an option and getopt
# will see it as an arg like config.yml
opts, args = getopt.gnu_getopt(sys.argv[1:], "gnqs:vx:h",
["google-drive", "no-copy", "quiet", "set", "help", "version"])
except getopt.GetoptError as err:
print_err(str(err))
usage()
sys.exit(2)
for o, a in opts:
if o in ("-g", "--google-drive"):
google_drive = True
elif o in ("-n", "--no-copy"):
do_copy = False
elif o in ("-q", "--quiet"):
quiet += 1
elif o in ("-s", "--set"):
try:
(arg, value) = a.split("=")
except ValueError as e:
print_err("{} should have the format `arg=value'".format(a))
sys.exit(2)
arg_set[arg] = value
elif o == "-v":
verbose += 1
elif o == "-x":
command = a
elif o in ("-h", "--help"):
usage()
sys.exit()
elif o == "--version":
show_version()
sys.exit()
else:
assert False, "unhandled option"
if not args:
smart_copy(None, arg_set, command, quiet, do_copy)
else:
for config_file in args:
if config_file == "-":
smart_copy(None, arg_set, command, quiet, do_copy)
else:
if os.path.exists(config_file):
smart_copy(config_file, arg_set, command, quiet, do_copy)
else:
print_err("{}: No such file or directory".format(config_file))
if __name__ == "__main__":
main()